From 9c7f4dbebfe01b5240d7e734845fa263fe2b8459 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Thu, 14 Nov 2024 16:11:09 +0100 Subject: [PATCH 01/31] Add Milvus skeleton --- milvus/CHANGELOG.md | 4 + milvus/README.md | 60 ++++++++++++ milvus/assets/configuration/spec.yaml | 10 ++ milvus/assets/dashboards/milvus_overview.json | 1 + milvus/assets/service_checks.json | 1 + milvus/changelog.d/1.added | 1 + milvus/datadog_checks/__init__.py | 4 + milvus/datadog_checks/milvus/__about__.py | 4 + milvus/datadog_checks/milvus/__init__.py | 7 ++ milvus/datadog_checks/milvus/check.py | 98 +++++++++++++++++++ .../milvus/config_models/__init__.py | 25 +++++ .../milvus/config_models/defaults.py | 16 +++ .../milvus/config_models/instance.py | 51 ++++++++++ .../milvus/config_models/shared.py | 48 +++++++++ .../milvus/config_models/validators.py | 13 +++ .../milvus/data/conf.yaml.example | 44 +++++++++ milvus/hatch.toml | 4 + milvus/images/IMAGES_README.md | 41 ++++++++ milvus/manifest.json | 57 +++++++++++ milvus/metadata.csv | 1 + milvus/pyproject.toml | 60 ++++++++++++ milvus/tests/__init__.py | 3 + milvus/tests/conftest.py | 14 +++ milvus/tests/test_unit.py | 26 +++++ 24 files changed, 593 insertions(+) create mode 100644 milvus/CHANGELOG.md create mode 100644 milvus/README.md create mode 100644 milvus/assets/configuration/spec.yaml create mode 100644 milvus/assets/dashboards/milvus_overview.json create mode 100644 milvus/assets/service_checks.json create mode 100644 milvus/changelog.d/1.added create mode 100644 milvus/datadog_checks/__init__.py create mode 100644 milvus/datadog_checks/milvus/__about__.py create mode 100644 milvus/datadog_checks/milvus/__init__.py create mode 100644 milvus/datadog_checks/milvus/check.py create mode 100644 milvus/datadog_checks/milvus/config_models/__init__.py create mode 100644 milvus/datadog_checks/milvus/config_models/defaults.py create mode 100644 milvus/datadog_checks/milvus/config_models/instance.py create mode 100644 milvus/datadog_checks/milvus/config_models/shared.py create mode 100644 milvus/datadog_checks/milvus/config_models/validators.py create mode 100644 milvus/datadog_checks/milvus/data/conf.yaml.example create mode 100644 milvus/hatch.toml create mode 100644 milvus/images/IMAGES_README.md create mode 100644 milvus/manifest.json create mode 100644 milvus/metadata.csv create mode 100644 milvus/pyproject.toml create mode 100644 milvus/tests/__init__.py create mode 100644 milvus/tests/conftest.py create mode 100644 milvus/tests/test_unit.py diff --git a/milvus/CHANGELOG.md b/milvus/CHANGELOG.md new file mode 100644 index 0000000000000..dd71dbd975e36 --- /dev/null +++ b/milvus/CHANGELOG.md @@ -0,0 +1,4 @@ +# CHANGELOG - Milvus + + + diff --git a/milvus/README.md b/milvus/README.md new file mode 100644 index 0000000000000..1a951270ac9ad --- /dev/null +++ b/milvus/README.md @@ -0,0 +1,60 @@ +# Agent Check: Milvus + +## Overview + +This check monitors [Milvus][1] through the Datadog Agent. + +Include a high level overview of what this integration does: +- What does your product do (in 1-2 sentences)? +- What value will customers get from this integration, and why is it valuable to them? +- What specific data will your integration monitor, and what's the value of that data? + +## Setup + +Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. + +### Installation + +The Milvus check is included in the [Datadog Agent][2] package. +No additional installation is needed on your server. + +### Configuration + +1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options. + +2. [Restart the Agent][5]. + +### Validation + +[Run the Agent's status subcommand][6] and look for `milvus` under the Checks section. + +## Data Collected + +### Metrics + +See [metadata.csv][7] for a list of metrics provided by this integration. + +### Events + +The Milvus integration does not include any events. + +### Service Checks + +The Milvus integration does not include any service checks. + +See [service_checks.json][8] for a list of service checks provided by this integration. + +## Troubleshooting + +Need help? Contact [Datadog support][9]. + + +[1]: **LINK_TO_INTEGRATION_SITE** +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/agent/kubernetes/integrations/ +[4]: https://github.com/DataDog/integrations-core/blob/master/milvus/datadog_checks/milvus/data/conf.yaml.example +[5]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent +[6]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[7]: https://github.com/DataDog/integrations-core/blob/master/milvus/metadata.csv +[8]: https://github.com/DataDog/integrations-core/blob/master/milvus/assets/service_checks.json +[9]: https://docs.datadoghq.com/help/ diff --git a/milvus/assets/configuration/spec.yaml b/milvus/assets/configuration/spec.yaml new file mode 100644 index 0000000000000..42c48c0a16a54 --- /dev/null +++ b/milvus/assets/configuration/spec.yaml @@ -0,0 +1,10 @@ +name: Milvus +files: +- name: milvus.yaml + options: + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - template: instances/default diff --git a/milvus/assets/dashboards/milvus_overview.json b/milvus/assets/dashboards/milvus_overview.json new file mode 100644 index 0000000000000..e9e23301af626 --- /dev/null +++ b/milvus/assets/dashboards/milvus_overview.json @@ -0,0 +1 @@ +Please build an out-of-the-box dashboard for your integration following our best practices here: https://datadoghq.dev/integrations-core/guidelines/dashboards/#best-practices \ No newline at end of file diff --git a/milvus/assets/service_checks.json b/milvus/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/milvus/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/milvus/changelog.d/1.added b/milvus/changelog.d/1.added new file mode 100644 index 0000000000000..aa949b47b7b41 --- /dev/null +++ b/milvus/changelog.d/1.added @@ -0,0 +1 @@ +Initial Release \ No newline at end of file diff --git a/milvus/datadog_checks/__init__.py b/milvus/datadog_checks/__init__.py new file mode 100644 index 0000000000000..1517d901c0aae --- /dev/null +++ b/milvus/datadog_checks/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/milvus/datadog_checks/milvus/__about__.py b/milvus/datadog_checks/milvus/__about__.py new file mode 100644 index 0000000000000..e9541ce83e9e5 --- /dev/null +++ b/milvus/datadog_checks/milvus/__about__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__version__ = '0.0.1' diff --git a/milvus/datadog_checks/milvus/__init__.py b/milvus/datadog_checks/milvus/__init__.py new file mode 100644 index 0000000000000..b0126262a42a2 --- /dev/null +++ b/milvus/datadog_checks/milvus/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .__about__ import __version__ +from .check import MilvusCheck + +__all__ = ['__version__', 'MilvusCheck'] diff --git a/milvus/datadog_checks/milvus/check.py b/milvus/datadog_checks/milvus/check.py new file mode 100644 index 0000000000000..509d04ceae6d5 --- /dev/null +++ b/milvus/datadog_checks/milvus/check.py @@ -0,0 +1,98 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any # noqa: F401 + +from datadog_checks.base import AgentCheck # noqa: F401 + +# from datadog_checks.base.utils.db import QueryManager +# from requests.exceptions import ConnectionError, HTTPError, InvalidURL, Timeout +# from json import JSONDecodeError + + +class MilvusCheck(AgentCheck): + + # This will be the prefix of every metric and service check the integration sends + __NAMESPACE__ = 'milvus' + + def __init__(self, name, init_config, instances): + super(MilvusCheck, self).__init__(name, init_config, instances) + + # Use self.instance to read the check configuration + # self.url = self.instance.get("url") + + # If the check is going to perform SQL queries you should define a query manager here. + # More info at + # https://datadoghq.dev/integrations-core/base/databases/#datadog_checks.base.utils.db.core.QueryManager + # sample_query = { + # "name": "sample", + # "query": "SELECT * FROM sample_table", + # "columns": [ + # {"name": "metric", "type": "gauge"} + # ], + # } + # self._query_manager = QueryManager(self, self.execute_query, queries=[sample_query]) + # self.check_initializations.append(self._query_manager.compile_queries) + + def check(self, _): + # type: (Any) -> None + # The following are useful bits of code to help new users get started. + + # Perform HTTP Requests with our HTTP wrapper. + # More info at https://datadoghq.dev/integrations-core/base/http/ + # try: + # response = self.http.get(self.url) + # response.raise_for_status() + # response_json = response.json() + + # except Timeout as e: + # self.service_check( + # "can_connect", + # AgentCheck.CRITICAL, + # message="Request timeout: {}, {}".format(self.url, e), + # ) + # raise + + # except (HTTPError, InvalidURL, ConnectionError) as e: + # self.service_check( + # "can_connect", + # AgentCheck.CRITICAL, + # message="Request failed: {}, {}".format(self.url, e), + # ) + # raise + + # except JSONDecodeError as e: + # self.service_check( + # "can_connect", + # AgentCheck.CRITICAL, + # message="JSON Parse failed: {}, {}".format(self.url, e), + # ) + # raise + + # except ValueError as e: + # self.service_check( + # "can_connect", AgentCheck.CRITICAL, message=str(e) + # ) + # raise + + # This is how you submit metrics + # There are different types of metrics that you can submit (gauge, event). + # More info at https://datadoghq.dev/integrations-core/base/api/#datadog_checks.base.checks.base.AgentCheck + # self.gauge("test", 1.23, tags=['foo:bar']) + + # Perform database queries using the Query Manager + # self._query_manager.execute() + + # This is how you use the persistent cache. This cache file based and persists across agent restarts. + # If you need an in-memory cache that is persisted across runs + # You can define a dictionary in the __init__ method. + # self.write_persistent_cache("key", "value") + # value = self.read_persistent_cache("key") + + # If your check ran successfully, you can send the status. + # More info at + # https://datadoghq.dev/integrations-core/base/api/#datadog_checks.base.checks.base.AgentCheck.service_check + # self.service_check("can_connect", AgentCheck.OK) + + # If it didn't then it should send a critical service check + self.service_check("can_connect", AgentCheck.CRITICAL) diff --git a/milvus/datadog_checks/milvus/config_models/__init__.py b/milvus/datadog_checks/milvus/config_models/__init__.py new file mode 100644 index 0000000000000..2e7eff5e94dd7 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/__init__.py @@ -0,0 +1,25 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/milvus/datadog_checks/milvus/config_models/defaults.py b/milvus/datadog_checks/milvus/config_models/defaults.py new file mode 100644 index 0000000000000..7adf7c7b7abd3 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/defaults.py @@ -0,0 +1,16 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +def instance_empty_default_hostname(): + return False + + +def instance_min_collection_interval(): + return 15 diff --git a/milvus/datadog_checks/milvus/config_models/instance.py b/milvus/datadog_checks/milvus/config_models/instance.py new file mode 100644 index 0000000000000..b2d32d2e62ea8 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/instance.py @@ -0,0 +1,51 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class InstanceConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + empty_default_hostname: Optional[bool] = None + min_collection_interval: Optional[float] = None + service: Optional[str] = None + tags: Optional[tuple[str, ...]] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) diff --git a/milvus/datadog_checks/milvus/config_models/shared.py b/milvus/datadog_checks/milvus/config_models/shared.py new file mode 100644 index 0000000000000..656c417aa3849 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/shared.py @@ -0,0 +1,48 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class SharedConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + service: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'shared_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) diff --git a/milvus/datadog_checks/milvus/config_models/validators.py b/milvus/datadog_checks/milvus/config_models/validators.py new file mode 100644 index 0000000000000..70150e85e6124 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/validators.py @@ -0,0 +1,13 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Here you can include additional config validators or transformers +# +# def initialize_instance(values, **kwargs): +# if 'my_option' not in values and 'my_legacy_option' in values: +# values['my_option'] = values['my_legacy_option'] +# if values.get('my_number') > 10: +# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) +# +# return values diff --git a/milvus/datadog_checks/milvus/data/conf.yaml.example b/milvus/datadog_checks/milvus/data/conf.yaml.example new file mode 100644 index 0000000000000..8ee633b1335fc --- /dev/null +++ b/milvus/datadog_checks/milvus/data/conf.yaml.example @@ -0,0 +1,44 @@ +## All options defined here are available to all instances. +# +init_config: + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# +instances: + + - + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 15 + ## This changes the collection interval of the check. For more information, see: + ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + # + # min_collection_interval: 15 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false diff --git a/milvus/hatch.toml b/milvus/hatch.toml new file mode 100644 index 0000000000000..001e43ce25414 --- /dev/null +++ b/milvus/hatch.toml @@ -0,0 +1,4 @@ +[env.collectors.datadog-checks] + +[[envs.default.matrix]] +python = ["3.11"] diff --git a/milvus/images/IMAGES_README.md b/milvus/images/IMAGES_README.md new file mode 100644 index 0000000000000..443f3c45e3385 --- /dev/null +++ b/milvus/images/IMAGES_README.md @@ -0,0 +1,41 @@ +# Marketplace Media Carousel Guidelines + +## Using the media gallery + +Please upload images to use the media gallery. Integrations require a minimum of 3 images. Images should highlight your product, your integration, and a full image of the Datadog integration dashboard. The gallery +can hold a maximum of 8 pieces of media total, and one of these pieces of media +can be a video (guidelines and submission steps below). Images should be +added to your /images directory and referenced in the manifest.json file. + + +## Image and video requirements + +### Images + +``` +File type : .jpg or .png +File size : ~500 KB per image, with a max of 1 MB per image +File dimensions : The image must be between 1440px and 2880px width, with a 16:9 aspect ratio (for example: 1440x810) +File name : Use only letters, numbers, underscores, and hyphens +Color mode : RGB +Color profile : sRGB +Description : 300 characters maximum +``` + +### Video + +To display a video in your media gallery, please send our team the zipped file +or a link to download the video at `marketplace@datadog.com`. In addition, +please upload a thumbnail image for your video as a part of the pull request. +Once approved, we will upload the file to Vimeo and provide you with the +vimeo_id to add to your manifest.json file. Please note that the gallery can +only hold one video. + +``` +File type : MP4 H.264 +File size : Max 1 video; 1 GB maximum size +File dimensions : The aspect ratio must be exactly 16:9, and the resolution must be 1920x1080 or higher +File name : partnerName-appName.mp4 +Run time : Recommendation of 60 seconds or less +Description : 300 characters maximum +``` diff --git a/milvus/manifest.json b/milvus/manifest.json new file mode 100644 index 0000000000000..d463b0f6c03ee --- /dev/null +++ b/milvus/manifest.json @@ -0,0 +1,57 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "38ddb395-6770-4b81-9730-e43cf4b4b2a0", + "app_id": "milvus", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "", + "title": "Milvus", + "media": [], + "classifier_tags": [ + "", + "Supported OS::Linux", + "Supported OS::Windows", + "Supported OS::macOS", + "Category::", + "Offering::", + "Queried Data Type::", + "Submitted Data Type::" + ] + }, + "assets": { + "integration": { + "auto_install": true, + "source_type_id": 30880529, + "source_type_name": "Milvus", + "configuration": { + "spec": "assets/configuration/spec.yaml" + }, + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "milvus.", + "check": "", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "": "assets/dashboards/.json" + }, + "monitors": {}, + "saved_views": {} + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/milvus/metadata.csv b/milvus/metadata.csv new file mode 100644 index 0000000000000..6bcfec7d94d0c --- /dev/null +++ b/milvus/metadata.csv @@ -0,0 +1 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric diff --git a/milvus/pyproject.toml b/milvus/pyproject.toml new file mode 100644 index 0000000000000..7f94dbf91c0d2 --- /dev/null +++ b/milvus/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = [ + "hatchling>=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-milvus" +description = "The Milvus check" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.11" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "milvus", +] +authors = [ + { name = "Datadog", email = "packages@datadoghq.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.11", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=32.6.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [] + +[project.urls] +Source = "https://github.com/DataDog/integrations-core" + +[tool.hatch.version] +path = "datadog_checks/milvus/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/milvus", +] +dev-mode-dirs = [ + ".", +] diff --git a/milvus/tests/__init__.py b/milvus/tests/__init__.py new file mode 100644 index 0000000000000..9103122bf028d --- /dev/null +++ b/milvus/tests/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/milvus/tests/conftest.py b/milvus/tests/conftest.py new file mode 100644 index 0000000000000..c8ac597a9862c --- /dev/null +++ b/milvus/tests/conftest.py @@ -0,0 +1,14 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + + +@pytest.fixture(scope='session') +def dd_environment(): + yield + + +@pytest.fixture +def instance(): + return {} diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py new file mode 100644 index 0000000000000..9d44e0f8919b5 --- /dev/null +++ b/milvus/tests/test_unit.py @@ -0,0 +1,26 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Any, Callable, Dict # noqa: F401 + +from datadog_checks.base import AgentCheck # noqa: F401 +from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 +from datadog_checks.dev.utils import get_metadata_metrics +from datadog_checks.milvus import MilvusCheck + + +def test_check(dd_run_check, aggregator, instance): + # type: (Callable[[AgentCheck, bool], None], AggregatorStub, Dict[str, Any]) -> None + check = MilvusCheck('milvus', {}, [instance]) + dd_run_check(check) + + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata(get_metadata_metrics()) + + +def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance): + # type: (Callable[[AgentCheck, bool], None], AggregatorStub, Dict[str, Any]) -> None + check = MilvusCheck('milvus', {}, [instance]) + dd_run_check(check) + aggregator.assert_service_check('milvus.can_connect', MilvusCheck.CRITICAL) From 838e6143b970d6c59745ca7094feef299eedcda1 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 18 Nov 2024 16:59:22 +0100 Subject: [PATCH 02/31] Add metrics to metadata.csv --- milvus/metadata.csv | 149 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 6bcfec7d94d0c..8d57c2526f020 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1 +1,150 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric +milvus_build_info,gauge,,,,Build information of milvus,0,milvus,, +milvus_cgo_active_future_total,gauge,,,,Total number of active futures.,0,milvus,, +milvus_cgo_cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus_cgo_cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, +milvus_cgo_running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, +milvus_datacoord_channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, +milvus_datacoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus_datacoord_consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus_datacoord_datanode_num,gauge,,,,number of data nodes,0,milvus,, +milvus_datacoord_import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, +milvus_datacoord_index_node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, +milvus_datacoord_index_req_count,count,,,,number of building index requests,0,milvus,, +milvus_datacoord_index_task_count,gauge,,,,number of index tasks of each type,0,milvus,, +milvus_datacoord_segment_num,gauge,,,,number of segments,0,milvus,, +milvus_datacoord_stored_binlog_size,gauge,,,,binlog size of healthy segments,0,milvus,, +milvus_datacoord_stored_index_files_size,gauge,,,,index files size of the segments,0,milvus,, +milvus_datacoord_stored_rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, +milvus_datacoord_task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, +milvus_datacoord_watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, +milvus_datanode_autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, +milvus_datanode_consume_bytes_count,count,,,,,0,milvus,, +milvus_datanode_consume_msg_count,count,,,,count of consumed msg,0,milvus,, +milvus_datanode_consume_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus_datanode_encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, +milvus_datanode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus_datanode_flush_buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, +milvus_datanode_flush_req_count,count,,,,count of flush request,0,milvus,, +milvus_datanode_flushed_data_rows,count,,,,num of rows flushed to storage,0,milvus,, +milvus_datanode_flushed_data_size,count,,,,byte size of data flushed to storage,0,milvus,, +milvus_datanode_msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus_datanode_msg_rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, +milvus_datanode_save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, +milvus_flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, +milvus_indexnode_build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, +milvus_indexnode_encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, +milvus_indexnode_index_task_count,count,,,,number of tasks that index node received,0,milvus,, +milvus_indexnode_index_task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, +milvus_indexnode_knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, +milvus_indexnode_save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, +milvus_meta_kv_size,gauge,,,,kv size stats,0,milvus,, +milvus_meta_op_count,count,,,,count of meta operation,0,milvus,, +milvus_meta_request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus_msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, +milvus_msgstream_op_count,count,,,,count of stream message operation,0,milvus,, +milvus_msgstream_request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus_num_node,gauge,,,,number of nodes and coordinates,0,milvus,, +milvus_proxy_apply_pk_latency,gauge,,,,latency that apply primary key,0,milvus,, +milvus_proxy_apply_timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, +milvus_proxy_assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus_proxy_cache_hit_count,count,,,,count of cache hits/miss,0,milvus,, +milvus_proxy_cache_update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, +milvus_proxy_delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, +milvus_proxy_msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, +milvus_proxy_mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus_proxy_rate_limit_req_count,count,,,,count of operation executed,0,milvus,, +milvus_proxy_report_value,count,,,,report value about the request,0,milvus,, +milvus_proxy_req_count,count,,,,count of operation executed,0,milvus,, +milvus_proxy_req_in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, +milvus_proxy_req_latency,gauge,,,,latency of each request,0,milvus,, +milvus_proxy_send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, +milvus_proxy_sq_decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, +milvus_proxy_sq_reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, +milvus_proxy_sq_wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, +milvus_proxy_sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus_proxy_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus_querycoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus_querycoord_current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, +milvus_querycoord_load_latency,gauge,,,,latency of load the entire collection,0,milvus,, +milvus_querycoord_load_req_count,count,,,,count of load request,0,milvus,, +milvus_querycoord_partition_num,gauge,,,,number of partitions,0,milvus,, +milvus_querycoord_querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, +milvus_querycoord_release_latency,gauge,,,,latency of release request,0,milvus,, +milvus_querycoord_release_req_count,count,,,,count of release request,0,milvus,, +milvus_querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, +milvus_querynode_apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, +milvus_querynode_collection_num,gauge,,,,number of collections loaded,0,milvus,, +milvus_querynode_consume_bytes_counter,count,,,,,0,milvus,, +milvus_querynode_consume_msg_count,count,,,,count of consumed msg,0,milvus,, +milvus_querynode_consume_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus_querynode_disk_cache_evict_bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, +milvus_querynode_disk_cache_evict_duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, +milvus_querynode_disk_cache_evict_global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, +milvus_querynode_disk_cache_evict_total,count,,,,number of segments evicted from disk cache,0,milvus,, +milvus_querynode_disk_cache_load_bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, +milvus_querynode_disk_cache_load_duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, +milvus_querynode_disk_cache_load_global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, +milvus_querynode_disk_cache_load_total,count,,,,number of segments loaded from disk cache,0,milvus,, +milvus_querynode_disk_used_size,gauge,,,,disk used size(MB),0,milvus,, +milvus_querynode_dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, +milvus_querynode_entity_num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, +milvus_querynode_entity_size,gauge,,,,"entities' memory size, clustered by collection and state",0,milvus,, +milvus_querynode_execute_bytes_counter,count,,,,,0,milvus,, +milvus_querynode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus_querynode_forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, +milvus_querynode_load_index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus_querynode_load_segment_concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, +milvus_querynode_load_segment_latency,gauge,,,,latency of load per segment,0,milvus,, +milvus_querynode_msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus_querynode_partition_num,gauge,,,,number of partitions loaded,0,milvus,, +milvus_querynode_process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, +milvus_querynode_read_task_concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, +milvus_querynode_read_task_ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, +milvus_querynode_read_task_unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, +milvus_querynode_search_group_nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, +milvus_querynode_search_group_size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, +milvus_querynode_search_group_topk,gauge,,,,the topK of each grouped search task,0,milvus,, +milvus_querynode_search_nq,gauge,,,,the number of queries of each search task,0,milvus,, +milvus_querynode_search_topk,gauge,,,,the top of each search task,0,milvus,, +milvus_querynode_segment_access_duration,count,,,,total time cost of accessing segments,0,milvus,, +milvus_querynode_segment_access_global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, +milvus_querynode_segment_access_total,count,,,,number of segments accessed,0,milvus,, +milvus_querynode_segment_access_wait_cache_duration,count,,,,total time cost of waiting for loading access,0,milvus,, +milvus_querynode_segment_access_wait_cache_global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, +milvus_querynode_segment_access_wait_cache_total,count,,,,number of segments waiting for loading access,0,milvus,, +milvus_querynode_segment_latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, +milvus_querynode_segment_num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, +milvus_querynode_sq_core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, +milvus_querynode_sq_queue_latency,gauge,,,,latency of search or query in queue,0,milvus,, +milvus_querynode_sq_queue_user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, +milvus_querynode_sq_reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, +milvus_querynode_sq_req_count,count,,,,count of search / query request,0,milvus,, +milvus_querynode_sq_req_latency,gauge,,,,latency of Search or query requests,0,milvus,, +milvus_querynode_sq_segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, +milvus_querynode_sq_wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, +milvus_querynode_wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, +milvus_querynode_watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, +milvus_rootcoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus_rootcoord_credential_num,gauge,,,,number of credentials,0,milvus,, +milvus_rootcoord_ddl_req_count,count,,,,count of DDL operations,0,milvus,, +milvus_rootcoord_ddl_req_latency,gauge,,,,latency of each DDL operations,0,milvus,, +milvus_rootcoord_ddl_req_latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, +milvus_rootcoord_dml_channel_num,gauge,,,,number of DML channels,0,milvus,, +milvus_rootcoord_entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, +milvus_rootcoord_force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, +milvus_rootcoord_id_alloc_count,count,,,,count of ID allocated,0,milvus,, +milvus_rootcoord_indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, +milvus_rootcoord_msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, +milvus_rootcoord_num_of_roles,gauge,,,,The number of roles,0,milvus,, +milvus_rootcoord_partition_num,gauge,,,,number of partitions,0,milvus,, +milvus_rootcoord_produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus_rootcoord_proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, +milvus_rootcoord_qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, +milvus_rootcoord_sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, +milvus_rootcoord_timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, +milvus_rootcoord_timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, +milvus_runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, +milvus_storage_kv_size,gauge,,,,kv size stats,0,milvus,, +milvus_storage_op_count,count,,,,count of persistent data operation,0,milvus,, +milvus_storage_request_latency,gauge,,,,request latency on the client side,0,milvus,, From 9bd81575d95e1a8f2a499619f9a02d3afaffb523 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 19 Nov 2024 15:11:07 +0100 Subject: [PATCH 03/31] Add periods in metadata.csv --- milvus/metadata.csv | 298 ++++++++++++++++++++++---------------------- 1 file changed, 149 insertions(+), 149 deletions(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 8d57c2526f020..fd98a468ec08e 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,150 +1,150 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -milvus_build_info,gauge,,,,Build information of milvus,0,milvus,, -milvus_cgo_active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus_cgo_cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus_cgo_cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, -milvus_cgo_running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, -milvus_datacoord_channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, -milvus_datacoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus_datacoord_consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus_datacoord_datanode_num,gauge,,,,number of data nodes,0,milvus,, -milvus_datacoord_import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, -milvus_datacoord_index_node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, -milvus_datacoord_index_req_count,count,,,,number of building index requests,0,milvus,, -milvus_datacoord_index_task_count,gauge,,,,number of index tasks of each type,0,milvus,, -milvus_datacoord_segment_num,gauge,,,,number of segments,0,milvus,, -milvus_datacoord_stored_binlog_size,gauge,,,,binlog size of healthy segments,0,milvus,, -milvus_datacoord_stored_index_files_size,gauge,,,,index files size of the segments,0,milvus,, -milvus_datacoord_stored_rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, -milvus_datacoord_task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, -milvus_datacoord_watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, -milvus_datanode_autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, -milvus_datanode_consume_bytes_count,count,,,,,0,milvus,, -milvus_datanode_consume_msg_count,count,,,,count of consumed msg,0,milvus,, -milvus_datanode_consume_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus_datanode_encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, -milvus_datanode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus_datanode_flush_buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, -milvus_datanode_flush_req_count,count,,,,count of flush request,0,milvus,, -milvus_datanode_flushed_data_rows,count,,,,num of rows flushed to storage,0,milvus,, -milvus_datanode_flushed_data_size,count,,,,byte size of data flushed to storage,0,milvus,, -milvus_datanode_msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus_datanode_msg_rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus_datanode_save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, -milvus_flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, -milvus_indexnode_build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, -milvus_indexnode_encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, -milvus_indexnode_index_task_count,count,,,,number of tasks that index node received,0,milvus,, -milvus_indexnode_index_task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, -milvus_indexnode_knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, -milvus_indexnode_save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, -milvus_meta_kv_size,gauge,,,,kv size stats,0,milvus,, -milvus_meta_op_count,count,,,,count of meta operation,0,milvus,, -milvus_meta_request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus_msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, -milvus_msgstream_op_count,count,,,,count of stream message operation,0,milvus,, -milvus_msgstream_request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus_num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus_proxy_apply_pk_latency,gauge,,,,latency that apply primary key,0,milvus,, -milvus_proxy_apply_timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, -milvus_proxy_assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus_proxy_cache_hit_count,count,,,,count of cache hits/miss,0,milvus,, -milvus_proxy_cache_update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, -milvus_proxy_delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, -milvus_proxy_msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus_proxy_mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus_proxy_rate_limit_req_count,count,,,,count of operation executed,0,milvus,, -milvus_proxy_report_value,count,,,,report value about the request,0,milvus,, -milvus_proxy_req_count,count,,,,count of operation executed,0,milvus,, -milvus_proxy_req_in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, -milvus_proxy_req_latency,gauge,,,,latency of each request,0,milvus,, -milvus_proxy_send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus_proxy_sq_decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, -milvus_proxy_sq_reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, -milvus_proxy_sq_wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, -milvus_proxy_sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus_proxy_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus_querycoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus_querycoord_current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, -milvus_querycoord_load_latency,gauge,,,,latency of load the entire collection,0,milvus,, -milvus_querycoord_load_req_count,count,,,,count of load request,0,milvus,, -milvus_querycoord_partition_num,gauge,,,,number of partitions,0,milvus,, -milvus_querycoord_querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus_querycoord_release_latency,gauge,,,,latency of release request,0,milvus,, -milvus_querycoord_release_req_count,count,,,,count of release request,0,milvus,, -milvus_querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus_querynode_apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, -milvus_querynode_collection_num,gauge,,,,number of collections loaded,0,milvus,, -milvus_querynode_consume_bytes_counter,count,,,,,0,milvus,, -milvus_querynode_consume_msg_count,count,,,,count of consumed msg,0,milvus,, -milvus_querynode_consume_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus_querynode_disk_cache_evict_bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, -milvus_querynode_disk_cache_evict_duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, -milvus_querynode_disk_cache_evict_global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, -milvus_querynode_disk_cache_evict_total,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus_querynode_disk_cache_load_bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, -milvus_querynode_disk_cache_load_duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, -milvus_querynode_disk_cache_load_global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, -milvus_querynode_disk_cache_load_total,count,,,,number of segments loaded from disk cache,0,milvus,, -milvus_querynode_disk_used_size,gauge,,,,disk used size(MB),0,milvus,, -milvus_querynode_dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, -milvus_querynode_entity_num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, -milvus_querynode_entity_size,gauge,,,,"entities' memory size, clustered by collection and state",0,milvus,, -milvus_querynode_execute_bytes_counter,count,,,,,0,milvus,, -milvus_querynode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus_querynode_forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, -milvus_querynode_load_index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus_querynode_load_segment_concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus_querynode_load_segment_latency,gauge,,,,latency of load per segment,0,milvus,, -milvus_querynode_msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus_querynode_partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus_querynode_process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, -milvus_querynode_read_task_concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, -milvus_querynode_read_task_ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, -milvus_querynode_read_task_unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, -milvus_querynode_search_group_nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, -milvus_querynode_search_group_size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, -milvus_querynode_search_group_topk,gauge,,,,the topK of each grouped search task,0,milvus,, -milvus_querynode_search_nq,gauge,,,,the number of queries of each search task,0,milvus,, -milvus_querynode_search_topk,gauge,,,,the top of each search task,0,milvus,, -milvus_querynode_segment_access_duration,count,,,,total time cost of accessing segments,0,milvus,, -milvus_querynode_segment_access_global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, -milvus_querynode_segment_access_total,count,,,,number of segments accessed,0,milvus,, -milvus_querynode_segment_access_wait_cache_duration,count,,,,total time cost of waiting for loading access,0,milvus,, -milvus_querynode_segment_access_wait_cache_global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, -milvus_querynode_segment_access_wait_cache_total,count,,,,number of segments waiting for loading access,0,milvus,, -milvus_querynode_segment_latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, -milvus_querynode_segment_num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus_querynode_sq_core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, -milvus_querynode_sq_queue_latency,gauge,,,,latency of search or query in queue,0,milvus,, -milvus_querynode_sq_queue_user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, -milvus_querynode_sq_reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, -milvus_querynode_sq_req_count,count,,,,count of search / query request,0,milvus,, -milvus_querynode_sq_req_latency,gauge,,,,latency of Search or query requests,0,milvus,, -milvus_querynode_sq_segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, -milvus_querynode_sq_wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, -milvus_querynode_wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, -milvus_querynode_watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, -milvus_rootcoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus_rootcoord_credential_num,gauge,,,,number of credentials,0,milvus,, -milvus_rootcoord_ddl_req_count,count,,,,count of DDL operations,0,milvus,, -milvus_rootcoord_ddl_req_latency,gauge,,,,latency of each DDL operations,0,milvus,, -milvus_rootcoord_ddl_req_latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, -milvus_rootcoord_dml_channel_num,gauge,,,,number of DML channels,0,milvus,, -milvus_rootcoord_entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, -milvus_rootcoord_force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, -milvus_rootcoord_id_alloc_count,count,,,,count of ID allocated,0,milvus,, -milvus_rootcoord_indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, -milvus_rootcoord_msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, -milvus_rootcoord_num_of_roles,gauge,,,,The number of roles,0,milvus,, -milvus_rootcoord_partition_num,gauge,,,,number of partitions,0,milvus,, -milvus_rootcoord_produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus_rootcoord_proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, -milvus_rootcoord_qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus_rootcoord_sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, -milvus_rootcoord_timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, -milvus_rootcoord_timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, -milvus_runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, -milvus_storage_kv_size,gauge,,,,kv size stats,0,milvus,, -milvus_storage_op_count,count,,,,count of persistent data operation,0,milvus,, -milvus_storage_request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, +milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, +milvus.cgo.cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, +milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, +milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, +milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, +milvus.datacoord.index.req_count,count,,,,number of building index requests,0,milvus,, +milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, +milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, +milvus.datacoord.stored.binlog_size,gauge,,,,binlog size of healthy segments,0,milvus,, +milvus.datacoord.stored.index_files_size,gauge,,,,index files size of the segments,0,milvus,, +milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, +milvus.datacoord.task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, +milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, +milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, +milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.datanode.consume.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.datanode.encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, +milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, +milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, +milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milvus,, +milvus.datanode.flushed_data.size,count,,,,byte size of data flushed to storage,0,milvus,, +milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, +milvus.datanode.save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, +milvus.flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, +milvus.indexnode.build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, +milvus.indexnode.encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, +milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, +milvus.indexnode.index.task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, +milvus.indexnode.knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, +milvus.meta.kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, +milvus.meta.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, +milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, +milvus.msgstream.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, +milvus.proxy.apply.pk_latency,gauge,,,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, +milvus.proxy.cache.update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, +milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, +milvus.proxy.mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, +milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, +milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, +milvus.proxy.req.in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.latency,gauge,,,,latency of each request,0,milvus,, +milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, +milvus.proxy.sq.decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.proxy.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, +milvus.querycoord.load.latency,gauge,,,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, +milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, +milvus.querycoord.release.latency,gauge,,,,latency of release request,0,milvus,, +milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, +milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, +milvus.querynode.apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, +milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, +milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, +milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.querynode.consume.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.querynode.disk.cache.evict.bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, +milvus.querynode.disk.used_size,gauge,,,,disk used size(MB),0,milvus,, +milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, +milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, +milvus.querynode.entity.size,gauge,,,,"entities' memory size, clustered by collection and state",0,milvus,, +milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, +milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.querynode.forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, +milvus.querynode.load.index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, +milvus.querynode.load.segment.latency,gauge,,,,latency of load per segment,0,milvus,, +milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, +milvus.querynode.process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, +milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, +milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, +milvus.querynode.search.group.nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, +milvus.querynode.search.group.size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.querynode.search.group.topk,gauge,,,,the topK of each grouped search task,0,milvus,, +milvus.querynode.search.nq,gauge,,,,the number of queries of each search task,0,milvus,, +milvus.querynode.search.topk,gauge,,,,the top of each search task,0,milvus,, +milvus.querynode.segment.access.duration,count,,,,total time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, +milvus.querynode.segment.access.wait_cache.duration,count,,,,total time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, +milvus.querynode.segment.latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, +milvus.querynode.sq.core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.queue.latency,gauge,,,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, +milvus.querynode.sq.req.latency,gauge,,,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, +milvus.querynode.watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, +milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, +milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency,gauge,,,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, +milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, +milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, +milvus.rootcoord.id_alloc_count,count,,,,count of ID allocated,0,milvus,, +milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, +milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, +milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, +milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, +milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, +milvus.rootcoord.sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, +milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, +milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, +milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, +milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, From bf868609bee3802a76410fba974c86d1e4db9ff8 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 19 Nov 2024 15:23:58 +0100 Subject: [PATCH 04/31] Update spec --- milvus/assets/configuration/spec.yaml | 17 +- .../milvus/data/conf.yaml.example | 584 +++++++++++++++++- 2 files changed, 594 insertions(+), 7 deletions(-) diff --git a/milvus/assets/configuration/spec.yaml b/milvus/assets/configuration/spec.yaml index 42c48c0a16a54..100b70fcc547f 100644 --- a/milvus/assets/configuration/spec.yaml +++ b/milvus/assets/configuration/spec.yaml @@ -2,9 +2,14 @@ name: Milvus files: - name: milvus.yaml options: - - template: init_config - options: - - template: init_config/default - - template: instances - options: - - template: instances/default + - template: init_config + options: + - template: init_config/openmetrics + - template: instances + options: + - template: instances/openmetrics + overrides: + openmetrics_endpoint.value.example: http://localhost:9091/metrics + openmetrics_endpoint.description: | + Endpoint exposing Milvus' Prometheus metrics. For more information, refer to + https://milvus.io/docs/monitor.md#Monitor-metrics-with-Prometheus. diff --git a/milvus/datadog_checks/milvus/data/conf.yaml.example b/milvus/datadog_checks/milvus/data/conf.yaml.example index 8ee633b1335fc..b275903aba352 100644 --- a/milvus/datadog_checks/milvus/data/conf.yaml.example +++ b/milvus/datadog_checks/milvus/data/conf.yaml.example @@ -2,6 +2,38 @@ # init_config: + ## @param proxy - mapping - optional + ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported like so: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## If set to `true`, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + + ## @param timeout - number - optional - default: 10 + ## The timeout for connecting to services. + # + # timeout: 10 + ## @param service - string - optional ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. ## @@ -13,7 +45,546 @@ init_config: # instances: - - + ## @param openmetrics_endpoint - string - required + ## Endpoint exposing Milvus' Prometheus metrics. For more information, refer to + ## https://milvus.io/docs/monitor.md#Monitor-metrics-with-Prometheus. + # + - openmetrics_endpoint: http://localhost:9091/metrics + + ## @param raw_metric_prefix - string - optional + ## A prefix that is removed from all exposed metric names, if present. + ## All configuration options will use the prefix-less name. + # + # raw_metric_prefix: _ + + ## @param extra_metrics - (list of string or mapping) - optional + ## This list defines metrics to collect from the `openmetrics_endpoint`, in addition to + ## what the check collects by default. If the check already collects a metric, then + ## metric definitions here take precedence. Metrics may be defined in 3 ways: + ## + ## 1. If the item is a string, then it represents the exposed metric name, and + ## the sent metric name will be identical. For example: + ## + ## extra_metrics: + ## - + ## - + ## 2. If the item is a mapping, then the keys represent the exposed metric names. + ## + ## a. If a value is a string, then it represents the sent metric name. For example: + ## + ## extra_metrics: + ## - : + ## - : + ## b. If a value is a mapping, then it must have a `name` and/or `type` key. + ## The `name` represents the sent metric name, and the `type` represents how + ## the metric should be handled, overriding any type information the endpoint + ## may provide. For example: + ## + ## extra_metrics: + ## - : + ## name: + ## type: + ## - : + ## name: + ## type: + ## + ## The supported native types are `gauge`, `counter`, `histogram`, and `summary`. + ## + ## Note: To collect counter metrics with names ending in `_total`, specify the metric name without the `_total` + ## suffix. For example, to collect the counter metric `promhttp_metric_handler_requests_total`, specify + ## `promhttp_metric_handler_requests`. This submits to Datadog the metric name appended with `.count`. + ## For more information, see: + ## https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes + ## + ## Regular expressions may be used to match the exposed metric names, for example: + ## + ## extra_metrics: + ## - ^network_(ingress|egress)_.+ + ## - .+: + ## type: gauge + # + # extra_metrics: [] + + ## @param exclude_metrics - list of strings - optional + ## A list of metrics to exclude, with each entry being either + ## the exact metric name or a regular expression. + ## In order to exclude all metrics but the ones matching a specific filter, + ## you can use a negative lookahead regex like: + ## - ^(?!foo).*$ + # + # exclude_metrics: [] + + ## @param exclude_metrics_by_labels - mapping - optional + ## A mapping of labels to exclude metrics with matching label name and their corresponding metric values. To match + ## all values of a label, set it to `true`. + ## + ## Note: Label filtering happens before `rename_labels`. + ## + ## For example, the following configuration instructs the check to exclude all metrics with + ## a label `worker` or a label `pid` with the value of either `23` or `42`. + ## + ## exclude_metrics_by_labels: + ## worker: true + ## pid: + ## - '23' + ## - '42' + # + # exclude_metrics_by_labels: {} + + ## @param exclude_labels - list of strings - optional + ## A list of labels to exclude, useful for high cardinality values like timestamps or UUIDs. + ## May be used in conjunction with `include_labels`. + ## Labels defined in `exclude_labels` will take precedence in case of overlap. + ## + ## Note: Label filtering happens before `rename_labels`. + # + # exclude_labels: [] + + ## @param include_labels - list of strings - optional + ## A list of labels to include. May be used in conjunction with `exclude_labels`. + ## Labels defined in `exclude_labels` will take precedence in case of overlap. + ## + ## Note: Label filtering happens before `rename_labels`. + # + # include_labels: [] + + ## @param rename_labels - mapping - optional + ## A mapping of label names to their new names. + # + # rename_labels: + # : + # : + + ## @param enable_health_service_check - boolean - optional - default: true + ## Whether or not to send a service check named `.openmetrics.health` which reports + ## the health of the `openmetrics_endpoint`. + # + # enable_health_service_check: true + + ## @param ignore_connection_errors - boolean - optional - default: false + ## Whether or not to ignore connection errors when scraping `openmetrics_endpoint`. + # + # ignore_connection_errors: false + + ## @param hostname_label - string - optional + ## Override the hostname for every metric submission with the value of one of its labels. + # + # hostname_label: + + ## @param hostname_format - string - optional + ## When `hostname_label` is set, this instructs the check how to format the values. The string + ## `` is replaced by the value of the label defined by `hostname_label`. + # + # hostname_format: + + ## @param collect_histogram_buckets - boolean - optional - default: true + ## Whether or not to send histogram buckets. + # + # collect_histogram_buckets: true + + ## @param non_cumulative_histogram_buckets - boolean - optional - default: false + ## Whether or not histogram buckets are non-cumulative and to come with a `lower_bound` tag. + # + # non_cumulative_histogram_buckets: false + + ## @param histogram_buckets_as_distributions - boolean - optional - default: false + ## Whether or not to send histogram buckets as Datadog distribution metrics. This implicitly + ## enables the `collect_histogram_buckets` and `non_cumulative_histogram_buckets` options. + ## + ## Learn more about distribution metrics: + ## https://docs.datadoghq.com/developers/metrics/types/?tab=distribution#metric-types + # + # histogram_buckets_as_distributions: false + + ## @param collect_counters_with_distributions - boolean - optional - default: false + ## Whether or not to also collect the observation counter metrics ending in `.sum` and `.count` + ## when sending histogram buckets as Datadog distribution metrics. This implicitly enables the + ## `histogram_buckets_as_distributions` option. + # + # collect_counters_with_distributions: false + + ## @param use_process_start_time - boolean - optional - default: false + ## Whether to enable a heuristic for reporting counter values on the first scrape. When true, + ## the first time an endpoint is scraped, check `process_start_time_seconds` to decide whether zero + ## initial value can be assumed for counters. This requires keeping metrics in memory until the entire + ## response is received. + # + # use_process_start_time: false + + ## @param share_labels - mapping - optional + ## This mapping allows for the sharing of labels across multiple metrics. The keys represent the + ## exposed metrics from which to share labels, and the values are mappings that configure the + ## sharing behavior. Each mapping must have at least one of the following keys: + ## + ## labels - This is a list of labels to share. All labels are shared if this is not set. + ## match - This is a list of labels to match on other metrics as a condition for sharing. + ## values - This is a list of allowed values as a condition for sharing. + ## + ## To unconditionally share all labels of a metric, set it to `true`. + ## + ## For example, the following configuration instructs the check to apply all labels from `metric_a` + ## to all other metrics, the `node` label from `metric_b` to only those metrics that have a `pod` + ## label value that matches the `pod` label value of `metric_b`, and all labels from `metric_c` + ## to all other metrics if their value is equal to `23` or `42`. + ## + ## share_labels: + ## metric_a: true + ## metric_b: + ## labels: + ## - node + ## match: + ## - pod + ## metric_c: + ## values: + ## - 23 + ## - 42 + # + # share_labels: {} + + ## @param cache_shared_labels - boolean - optional - default: true + ## When `share_labels` is set, it instructs the check to cache labels collected from the first payload + ## for improved performance. + ## + ## Set this to `false` to compute label sharing for every payload at the risk of potentially increased memory usage. + # + # cache_shared_labels: true + + ## @param raw_line_filters - list of strings - optional + ## A list of regular expressions used to exclude lines read from the `openmetrics_endpoint` + ## from being parsed. + # + # raw_line_filters: [] + + ## @param cache_metric_wildcards - boolean - optional - default: true + ## Whether or not to cache data from metrics that are defined by regular expressions rather + ## than the full metric name. + # + # cache_metric_wildcards: true + + ## @param telemetry - boolean - optional - default: false + ## Whether or not to submit metrics prefixed by `.telemetry.` for debugging purposes. + # + # telemetry: false + + ## @param ignore_tags - list of strings - optional + ## A list of regular expressions used to ignore tags added by Autodiscovery and entries in the `tags` option. + # + # ignore_tags: + # - + # - + # - + + ## @param proxy - mapping - optional + ## This overrides the `proxy` setting in `init_config`. + ## + ## Set HTTP or HTTPS proxies for this instance. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported, for example: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## This overrides the `skip_proxy` setting in `init_config`. + ## + ## If set to `true`, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + + ## @param auth_type - string - optional - default: basic + ## The type of authentication to use. The available types (and related options) are: + ## + ## - basic + ## |__ username + ## |__ password + ## |__ use_legacy_auth_encoding + ## - digest + ## |__ username + ## |__ password + ## - ntlm + ## |__ ntlm_domain + ## |__ password + ## - kerberos + ## |__ kerberos_auth + ## |__ kerberos_cache + ## |__ kerberos_delegate + ## |__ kerberos_force_initiate + ## |__ kerberos_hostname + ## |__ kerberos_keytab + ## |__ kerberos_principal + ## - aws + ## |__ aws_region + ## |__ aws_host + ## |__ aws_service + ## + ## The `aws` auth type relies on boto3 to automatically gather AWS credentials, for example: from `.aws/credentials`. + ## Details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials + # + # auth_type: basic + + ## @param use_legacy_auth_encoding - boolean - optional - default: true + ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`. + # + # use_legacy_auth_encoding: true + + ## @param username - string - optional + ## The username to use if services are behind basic or digest auth. + # + # username: + + ## @param password - string - optional + ## The password to use if services are behind basic or NTLM auth. + # + # password: + + ## @param ntlm_domain - string - optional + ## If your services use NTLM authentication, specify + ## the domain used in the check. For NTLM Auth, append + ## the username to domain, not as the `username` parameter. + # + # ntlm_domain: \ + + ## @param kerberos_auth - string - optional - default: disabled + ## If your services use Kerberos authentication, you can specify the Kerberos + ## strategy to use between: + ## + ## - required + ## - optional + ## - disabled + ## + ## See https://github.com/requests/requests-kerberos#mutual-authentication + # + # kerberos_auth: disabled + + ## @param kerberos_cache - string - optional + ## Sets the KRB5CCNAME environment variable. + ## It should point to a credential cache with a valid TGT. + # + # kerberos_cache: + + ## @param kerberos_delegate - boolean - optional - default: false + ## Set to `true` to enable Kerberos delegation of credentials to a server that requests delegation. + ## + ## See https://github.com/requests/requests-kerberos#delegation + # + # kerberos_delegate: false + + ## @param kerberos_force_initiate - boolean - optional - default: false + ## Set to `true` to preemptively initiate the Kerberos GSS exchange and + ## present a Kerberos ticket on the initial request (and all subsequent). + ## + ## See https://github.com/requests/requests-kerberos#preemptive-authentication + # + # kerberos_force_initiate: false + + ## @param kerberos_hostname - string - optional + ## Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't + ## match its Kerberos hostname, for example: behind a content switch or load balancer. + ## + ## See https://github.com/requests/requests-kerberos#hostname-override + # + # kerberos_hostname: + + ## @param kerberos_principal - string - optional + ## Set an explicit principal, to force Kerberos to look for a + ## matching credential cache for the named user. + ## + ## See https://github.com/requests/requests-kerberos#explicit-principal + # + # kerberos_principal: + + ## @param kerberos_keytab - string - optional + ## Set the path to your Kerberos key tab file. + # + # kerberos_keytab: + + ## @param auth_token - mapping - optional + ## This allows for the use of authentication information from dynamic sources. + ## Both a reader and writer must be configured. + ## + ## The available readers are: + ## + ## - type: file + ## path (required): The absolute path for the file to read from. + ## pattern: A regular expression pattern with a single capture group used to find the + ## token rather than using the entire file, for example: Your secret is (.+) + ## - type: oauth + ## url (required): The token endpoint. + ## client_id (required): The client identifier. + ## client_secret (required): The client secret. + ## basic_auth: Whether the provider expects credentials to be transmitted in + ## an HTTP Basic Auth header. The default is: false + ## options: Mapping of additional options to pass to the provider, such as the audience + ## or the scope. For example: + ## options: + ## audience: https://example.com + ## scope: read:example + ## + ## The available writers are: + ## + ## - type: header + ## name (required): The name of the field, for example: Authorization + ## value: The template value, for example `Bearer `. The default is: + ## placeholder: The substring in `value` to replace with the token, defaults to: + # + # auth_token: + # reader: + # type: + # : + # : + # writer: + # type: + # : + # : + + ## @param aws_region - string - optional + ## If your services require AWS Signature Version 4 signing, set the region. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_region: + + ## @param aws_host - string - optional + ## If your services require AWS Signature Version 4 signing, set the host. + ## This only needs the hostname and does not require the protocol (HTTP, HTTPS, and more). + ## For example, if connecting to https://us-east-1.amazonaws.com/, set `aws_host` to `us-east-1.amazonaws.com`. + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_host: + + ## @param aws_service - string - optional + ## If your services require AWS Signature Version 4 signing, set the service code. For a list + ## of available service codes, see https://docs.aws.amazon.com/general/latest/gr/rande.html + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_service: + + ## @param tls_verify - boolean - optional - default: true + ## Instructs the check to validate the TLS certificate of services. + # + # tls_verify: true + + ## @param tls_use_host_header - boolean - optional - default: false + ## If a `Host` header is set, this enables its use for SNI (matching against the TLS certificate CN or SAN). + # + # tls_use_host_header: false + + ## @param tls_ignore_warning - boolean - optional - default: false + ## If `tls_verify` is disabled, security warnings are logged by the check. + ## Disable those by setting `tls_ignore_warning` to true. + # + # tls_ignore_warning: false + + ## @param tls_cert - string - optional + ## The path to a single file in PEM format containing a certificate as well as any + ## number of CA certificates needed to establish the certificate's authenticity for + ## use when connecting to services. It may also contain an unencrypted private key to use. + # + # tls_cert: + + ## @param tls_private_key - string - optional + ## The unencrypted private key to use for `tls_cert` when connecting to services. This is + ## required if `tls_cert` is set and it does not already contain a private key. + # + # tls_private_key: + + ## @param tls_ca_cert - string - optional + ## The path to a file of concatenated CA certificates in PEM format or a directory + ## containing several CA certificates in PEM format. If a directory, the directory + ## must have been processed using the `openssl rehash` command. See: + ## https://www.openssl.org/docs/man3.2/man1/c_rehash.html + # + # tls_ca_cert: + + ## @param tls_protocols_allowed - list of strings - optional + ## The expected versions of TLS/SSL when fetching intermediate certificates. + ## Only `SSLv3`, `TLSv1.2`, `TLSv1.3` are allowed by default. The possible values are: + ## SSLv3 + ## TLSv1 + ## TLSv1.1 + ## TLSv1.2 + ## TLSv1.3 + # + # tls_protocols_allowed: + # - SSLv3 + # - TLSv1.2 + # - TLSv1.3 + + ## @param headers - mapping - optional + ## The headers parameter allows you to send specific headers with every request. + ## You can use it for explicitly specifying the host header or adding headers for + ## authorization purposes. + ## + ## This overrides any default headers. + # + # headers: + # Host: + # X-Auth-Token: + + ## @param extra_headers - mapping - optional + ## Additional headers to send with every request. + # + # extra_headers: + # Host: + # X-Auth-Token: + + ## @param timeout - number - optional - default: 10 + ## The timeout for accessing services. + ## + ## This overrides the `timeout` setting in `init_config`. + # + # timeout: 10 + + ## @param connect_timeout - number - optional + ## The connect timeout for accessing services. Defaults to `timeout`. + # + # connect_timeout: + + ## @param read_timeout - number - optional + ## The read timeout for accessing services. Defaults to `timeout`. + # + # read_timeout: + + ## @param request_size - number - optional - default: 16 + ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time. + # + # request_size: 16 + + ## @param log_requests - boolean - optional - default: false + ## Whether or not to debug log the HTTP(S) requests made, including the method and URL. + # + # log_requests: false + + ## @param persist_connections - boolean - optional - default: false + ## Whether or not to persist cookies and use connection pooling for improved performance. + # + # persist_connections: false + + ## @param allow_redirects - boolean - optional - default: true + ## Whether or not to allow URL redirection. + # + # allow_redirects: true + ## @param tags - list of strings - optional ## A list of tags to attach to every metric and service check emitted by this instance. ## @@ -42,3 +613,14 @@ instances: ## This is useful for cluster-level checks. # # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - From 18c16ac29f2751ccf7a0c265a9416692c8e13b25 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 19 Nov 2024 15:26:38 +0100 Subject: [PATCH 05/31] Update config models --- .../milvus/config_models/__init__.py | 1 - .../milvus/config_models/defaults.py | 116 ++++++++++++++++ .../milvus/config_models/instance.py | 126 +++++++++++++++++- .../milvus/config_models/shared.py | 14 +- 4 files changed, 252 insertions(+), 5 deletions(-) diff --git a/milvus/datadog_checks/milvus/config_models/__init__.py b/milvus/datadog_checks/milvus/config_models/__init__.py index 2e7eff5e94dd7..106fff2032f68 100644 --- a/milvus/datadog_checks/milvus/config_models/__init__.py +++ b/milvus/datadog_checks/milvus/config_models/__init__.py @@ -7,7 +7,6 @@ # ddev -x validate config -s # ddev -x validate models -s - from .instance import InstanceConfig from .shared import SharedConfig diff --git a/milvus/datadog_checks/milvus/config_models/defaults.py b/milvus/datadog_checks/milvus/config_models/defaults.py index 7adf7c7b7abd3..bf7519af75f42 100644 --- a/milvus/datadog_checks/milvus/config_models/defaults.py +++ b/milvus/datadog_checks/milvus/config_models/defaults.py @@ -8,9 +8,125 @@ # ddev -x validate models -s +def shared_skip_proxy(): + return False + + +def shared_timeout(): + return 10 + + +def instance_allow_redirects(): + return True + + +def instance_auth_type(): + return 'basic' + + +def instance_cache_metric_wildcards(): + return True + + +def instance_cache_shared_labels(): + return True + + +def instance_collect_counters_with_distributions(): + return False + + +def instance_collect_histogram_buckets(): + return True + + +def instance_disable_generic_tags(): + return False + + def instance_empty_default_hostname(): return False +def instance_enable_health_service_check(): + return True + + +def instance_histogram_buckets_as_distributions(): + return False + + +def instance_ignore_connection_errors(): + return False + + +def instance_kerberos_auth(): + return 'disabled' + + +def instance_kerberos_delegate(): + return False + + +def instance_kerberos_force_initiate(): + return False + + +def instance_log_requests(): + return False + + def instance_min_collection_interval(): return 15 + + +def instance_non_cumulative_histogram_buckets(): + return False + + +def instance_persist_connections(): + return False + + +def instance_request_size(): + return 16 + + +def instance_skip_proxy(): + return False + + +def instance_tag_by_endpoint(): + return True + + +def instance_telemetry(): + return False + + +def instance_timeout(): + return 10 + + +def instance_tls_ignore_warning(): + return False + + +def instance_tls_use_host_header(): + return False + + +def instance_tls_verify(): + return True + + +def instance_use_latest_spec(): + return False + + +def instance_use_legacy_auth_encoding(): + return True + + +def instance_use_process_start_time(): + return False diff --git a/milvus/datadog_checks/milvus/config_models/instance.py b/milvus/datadog_checks/milvus/config_models/instance.py index b2d32d2e62ea8..8e39a0e921719 100644 --- a/milvus/datadog_checks/milvus/config_models/instance.py +++ b/milvus/datadog_checks/milvus/config_models/instance.py @@ -7,12 +7,12 @@ # ddev -x validate config -s # ddev -x validate models -s - from __future__ import annotations -from typing import Optional +from types import MappingProxyType +from typing import Any, Optional, Union -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation @@ -20,16 +20,136 @@ from . import defaults, validators +class AuthToken(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + reader: Optional[MappingProxyType[str, Any]] = None + writer: Optional[MappingProxyType[str, Any]] = None + + +class ExtraMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra='allow', + frozen=True, + ) + name: Optional[str] = None + type: Optional[str] = None + + +class MetricPatterns(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + + +class Metrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra='allow', + frozen=True, + ) + name: Optional[str] = None + type: Optional[str] = None + + +class Proxy(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + http: Optional[str] = None + https: Optional[str] = None + no_proxy: Optional[tuple[str, ...]] = None + + +class ShareLabels(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + labels: Optional[tuple[str, ...]] = None + match: Optional[tuple[str, ...]] = None + + class InstanceConfig(BaseModel): model_config = ConfigDict( validate_default=True, arbitrary_types_allowed=True, frozen=True, ) + allow_redirects: Optional[bool] = None + auth_token: Optional[AuthToken] = None + auth_type: Optional[str] = None + aws_host: Optional[str] = None + aws_region: Optional[str] = None + aws_service: Optional[str] = None + cache_metric_wildcards: Optional[bool] = None + cache_shared_labels: Optional[bool] = None + collect_counters_with_distributions: Optional[bool] = None + collect_histogram_buckets: Optional[bool] = None + connect_timeout: Optional[float] = None + disable_generic_tags: Optional[bool] = None empty_default_hostname: Optional[bool] = None + enable_health_service_check: Optional[bool] = None + exclude_labels: Optional[tuple[str, ...]] = None + exclude_metrics: Optional[tuple[str, ...]] = None + exclude_metrics_by_labels: Optional[MappingProxyType[str, Union[bool, tuple[str, ...]]]] = None + extra_headers: Optional[MappingProxyType[str, Any]] = None + extra_metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, ExtraMetrics]]], ...]] = None + headers: Optional[MappingProxyType[str, Any]] = None + histogram_buckets_as_distributions: Optional[bool] = None + hostname_format: Optional[str] = None + hostname_label: Optional[str] = None + ignore_connection_errors: Optional[bool] = None + ignore_tags: Optional[tuple[str, ...]] = None + include_labels: Optional[tuple[str, ...]] = None + kerberos_auth: Optional[str] = None + kerberos_cache: Optional[str] = None + kerberos_delegate: Optional[bool] = None + kerberos_force_initiate: Optional[bool] = None + kerberos_hostname: Optional[str] = None + kerberos_keytab: Optional[str] = None + kerberos_principal: Optional[str] = None + log_requests: Optional[bool] = None + metric_patterns: Optional[MetricPatterns] = None + metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, Metrics]]], ...]] = None min_collection_interval: Optional[float] = None + namespace: Optional[str] = Field(None, pattern='\\w*') + non_cumulative_histogram_buckets: Optional[bool] = None + ntlm_domain: Optional[str] = None + openmetrics_endpoint: str + password: Optional[str] = None + persist_connections: Optional[bool] = None + proxy: Optional[Proxy] = None + raw_line_filters: Optional[tuple[str, ...]] = None + raw_metric_prefix: Optional[str] = None + read_timeout: Optional[float] = None + rename_labels: Optional[MappingProxyType[str, Any]] = None + request_size: Optional[float] = None service: Optional[str] = None + share_labels: Optional[MappingProxyType[str, Union[bool, ShareLabels]]] = None + skip_proxy: Optional[bool] = None + tag_by_endpoint: Optional[bool] = None tags: Optional[tuple[str, ...]] = None + telemetry: Optional[bool] = None + timeout: Optional[float] = None + tls_ca_cert: Optional[str] = None + tls_cert: Optional[str] = None + tls_ignore_warning: Optional[bool] = None + tls_private_key: Optional[str] = None + tls_protocols_allowed: Optional[tuple[str, ...]] = None + tls_use_host_header: Optional[bool] = None + tls_verify: Optional[bool] = None + use_latest_spec: Optional[bool] = None + use_legacy_auth_encoding: Optional[bool] = None + use_process_start_time: Optional[bool] = None + username: Optional[str] = None @model_validator(mode='before') def _initial_validation(cls, values): diff --git a/milvus/datadog_checks/milvus/config_models/shared.py b/milvus/datadog_checks/milvus/config_models/shared.py index 656c417aa3849..0e8a9ecab10a2 100644 --- a/milvus/datadog_checks/milvus/config_models/shared.py +++ b/milvus/datadog_checks/milvus/config_models/shared.py @@ -7,7 +7,6 @@ # ddev -x validate config -s # ddev -x validate models -s - from __future__ import annotations from typing import Optional @@ -20,13 +19,26 @@ from . import defaults, validators +class Proxy(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + http: Optional[str] = None + https: Optional[str] = None + no_proxy: Optional[tuple[str, ...]] = None + + class SharedConfig(BaseModel): model_config = ConfigDict( validate_default=True, arbitrary_types_allowed=True, frozen=True, ) + proxy: Optional[Proxy] = None service: Optional[str] = None + skip_proxy: Optional[bool] = None + timeout: Optional[float] = None @model_validator(mode='before') def _initial_validation(cls, values): From d3d292c1321ed26d0d3cc9277aba45263eed807b Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 19 Nov 2024 15:33:26 +0100 Subject: [PATCH 06/31] Update manifest --- milvus/manifest.json | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/milvus/manifest.json b/milvus/manifest.json index d463b0f6c03ee..44a7ffd9b74be 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -12,14 +12,15 @@ "title": "Milvus", "media": [], "classifier_tags": [ - "", "Supported OS::Linux", "Supported OS::Windows", "Supported OS::macOS", - "Category::", - "Offering::", - "Queried Data Type::", - "Submitted Data Type::" + "Category::AI", + "Category::Data Store", + "Category::Machine Learning & LLM", + "Offering::Integration", + "Submitted Data Type::Metrics", + "Submitted Data Type::Logs" ] }, "assets": { @@ -41,12 +42,7 @@ "service_checks": { "metadata_path": "assets/service_checks.json" } - }, - "dashboards": { - "": "assets/dashboards/.json" - }, - "monitors": {}, - "saved_views": {} + } }, "author": { "support_email": "help@datadoghq.com", From e20c23b11977f3509792cbc83e71f781ed27818d Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 19 Nov 2024 15:37:31 +0100 Subject: [PATCH 07/31] Update check.py --- milvus/datadog_checks/milvus/check.py | 79 +-------------------------- 1 file changed, 2 insertions(+), 77 deletions(-) diff --git a/milvus/datadog_checks/milvus/check.py b/milvus/datadog_checks/milvus/check.py index 509d04ceae6d5..cc3612d85463a 100644 --- a/milvus/datadog_checks/milvus/check.py +++ b/milvus/datadog_checks/milvus/check.py @@ -3,14 +3,14 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from typing import Any # noqa: F401 -from datadog_checks.base import AgentCheck # noqa: F401 +from datadog_checks.base import OpenMetricsBaseCheckV2 # from datadog_checks.base.utils.db import QueryManager # from requests.exceptions import ConnectionError, HTTPError, InvalidURL, Timeout # from json import JSONDecodeError -class MilvusCheck(AgentCheck): +class MilvusCheck(OpenMetricsBaseCheckV2): # This will be the prefix of every metric and service check the integration sends __NAMESPACE__ = 'milvus' @@ -21,78 +21,3 @@ def __init__(self, name, init_config, instances): # Use self.instance to read the check configuration # self.url = self.instance.get("url") - # If the check is going to perform SQL queries you should define a query manager here. - # More info at - # https://datadoghq.dev/integrations-core/base/databases/#datadog_checks.base.utils.db.core.QueryManager - # sample_query = { - # "name": "sample", - # "query": "SELECT * FROM sample_table", - # "columns": [ - # {"name": "metric", "type": "gauge"} - # ], - # } - # self._query_manager = QueryManager(self, self.execute_query, queries=[sample_query]) - # self.check_initializations.append(self._query_manager.compile_queries) - - def check(self, _): - # type: (Any) -> None - # The following are useful bits of code to help new users get started. - - # Perform HTTP Requests with our HTTP wrapper. - # More info at https://datadoghq.dev/integrations-core/base/http/ - # try: - # response = self.http.get(self.url) - # response.raise_for_status() - # response_json = response.json() - - # except Timeout as e: - # self.service_check( - # "can_connect", - # AgentCheck.CRITICAL, - # message="Request timeout: {}, {}".format(self.url, e), - # ) - # raise - - # except (HTTPError, InvalidURL, ConnectionError) as e: - # self.service_check( - # "can_connect", - # AgentCheck.CRITICAL, - # message="Request failed: {}, {}".format(self.url, e), - # ) - # raise - - # except JSONDecodeError as e: - # self.service_check( - # "can_connect", - # AgentCheck.CRITICAL, - # message="JSON Parse failed: {}, {}".format(self.url, e), - # ) - # raise - - # except ValueError as e: - # self.service_check( - # "can_connect", AgentCheck.CRITICAL, message=str(e) - # ) - # raise - - # This is how you submit metrics - # There are different types of metrics that you can submit (gauge, event). - # More info at https://datadoghq.dev/integrations-core/base/api/#datadog_checks.base.checks.base.AgentCheck - # self.gauge("test", 1.23, tags=['foo:bar']) - - # Perform database queries using the Query Manager - # self._query_manager.execute() - - # This is how you use the persistent cache. This cache file based and persists across agent restarts. - # If you need an in-memory cache that is persisted across runs - # You can define a dictionary in the __init__ method. - # self.write_persistent_cache("key", "value") - # value = self.read_persistent_cache("key") - - # If your check ran successfully, you can send the status. - # More info at - # https://datadoghq.dev/integrations-core/base/api/#datadog_checks.base.checks.base.AgentCheck.service_check - # self.service_check("can_connect", AgentCheck.OK) - - # If it didn't then it should send a critical service check - self.service_check("can_connect", AgentCheck.CRITICAL) From 9486b21e96e965bcfae9cf1e4f68ac9bc9d9a06a Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 25 Nov 2024 13:30:11 +0100 Subject: [PATCH 08/31] Add process signature --- milvus/manifest.json | 5 ++++- milvus/tests/test_unit.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/milvus/manifest.json b/milvus/manifest.json index 44a7ffd9b74be..0f16fb61c2faa 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -41,7 +41,10 @@ }, "service_checks": { "metadata_path": "assets/service_checks.json" - } + }, + "process_signatures": [ + "milvus" + ] } }, "author": { diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py index 9d44e0f8919b5..fbf965dd4ca18 100644 --- a/milvus/tests/test_unit.py +++ b/milvus/tests/test_unit.py @@ -4,9 +4,11 @@ from typing import Any, Callable, Dict # noqa: F401 +import pytest from datadog_checks.base import AgentCheck # noqa: F401 from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 from datadog_checks.dev.utils import get_metadata_metrics + from datadog_checks.milvus import MilvusCheck @@ -24,3 +26,21 @@ def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggrega check = MilvusCheck('milvus', {}, [instance]) dd_run_check(check) aggregator.assert_service_check('milvus.can_connect', MilvusCheck.CRITICAL) + +def test_empty_instance(dd_run_check): + with pytest.raises( + Exception, + match='InstanceConfig`:\nopenmetrics_endpoint\n Field required', + ): + check = MilvusCheck('aws_neuron', {}, [{}]) + dd_run_check(check) + + +def test_custom_validation(dd_run_check): + endpoint = 'aws_neuron:2112/metrics' + with pytest.raises( + Exception, + match='openmetrics_endpoint: {} is incorrectly configured'.format(endpoint), + ): + check = MilvusCheck('aws_neuron', {}, [{'openmetrics_endpoint': endpoint}]) + dd_run_check(check) From aa710431d7f0617f0caa91cadc28df0a651dc56d Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 25 Nov 2024 15:03:46 +0100 Subject: [PATCH 09/31] Add units in metadata.csv --- milvus/metadata.csv | 130 ++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index fd98a468ec08e..c21b8ddae17a7 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,104 +1,104 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus.cgo.cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.cgo_duration_seconds,gauge,,second,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds,gauge,,second,,Duration of cgo call in queue.,0,milvus,, milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, milvus.datacoord.index.req_count,count,,,,number of building index requests,0,milvus,, milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, -milvus.datacoord.stored.binlog_size,gauge,,,,binlog size of healthy segments,0,milvus,, -milvus.datacoord.stored.index_files_size,gauge,,,,index files size of the segments,0,milvus,, +milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, +milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,, milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, -milvus.datacoord.task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, +milvus.datacoord.task_execute_max_latency,gauge,,millisecond,,latency of task execute operation,0,milvus,, milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, -milvus.datanode.consume.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus.datanode.encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.datanode.encode_buffer_latency,gauge,,millisecond,,latency of encode buffer data,0,milvus,, milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milvus,, -milvus.datanode.flushed_data.size,count,,,,byte size of data flushed to storage,0,milvus,, -milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.datanode.flushed_data.size,count,,byte,,byte size of data flushed to storage,0,milvus,, +milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus.datanode.save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, +milvus.datanode.save_latency,gauge,,millisecond,,latency of saving flush data to storage,0,milvus,, milvus.flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, -milvus.indexnode.build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, -milvus.indexnode.encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, +milvus.indexnode.build_index_latency,gauge,,millisecond,,latency of build index for segment,0,milvus,, +milvus.indexnode.encode_index_latency,gauge,,millisecond,,latency of encoding the index file,0,milvus,, milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, -milvus.indexnode.index.task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, -milvus.indexnode.knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, +milvus.indexnode.index.task_latency_in_queue,gauge,,millisecond,,latency of index task in queue,0,milvus,, +milvus.indexnode.knowhere_build_index_latency,gauge,,millisecond,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.save_index_latency,gauge,,millisecond,,latency of saving the index file,0,milvus,, milvus.meta.kv_size,gauge,,,,kv size stats,0,milvus,, milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, -milvus.meta.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.meta.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, -milvus.msgstream.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.msgstream.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus.proxy.apply.pk_latency,gauge,,,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.apply.pk_latency,gauge,,millisecond,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.timestamp_latency,gauge,,millisecond,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.assign_segmentID_latency,gauge,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, -milvus.proxy.cache.update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.cache.update_latency,gauge,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus.proxy.mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.mutation_send_latency,gauge,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.req.in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.latency,gauge,,,,latency of each request,0,milvus,, +milvus.proxy.req.in_queue_latency,gauge,,millisecond,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.latency,gauge,,millisecond,,latency of each request,0,milvus,, milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus.proxy.sq.decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sq.decode_result_latency,gauge,,millisecond,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency,gauge,,millisecond,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.wait_result_latency,gauge,,millisecond,,latency that proxy waits for the result,0,milvus,, milvus.proxy.sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.proxy.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, -milvus.querycoord.load.latency,gauge,,,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.latency,gauge,,millisecond,,latency of load the entire collection,0,milvus,, milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus.querycoord.release.latency,gauge,,,,latency of release request,0,milvus,, +milvus.querycoord.release.latency,gauge,,millisecond,,latency of release request,0,milvus,, milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus.querynode.apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, +milvus.querynode.apply_bf_latency,gauge,,millisecond,,apply bf cost in ms,0,milvus,, milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, -milvus.querynode.consume.tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus.querynode.disk.cache.evict.bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.querynode.disk.cache.evict.bytes,count,,byte,,number of bytes evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.duration,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration,gauge,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.bytes,count,,byte,,number of bytes loaded from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.duration,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration,gauge,,millisecond,,global duration of loading segments from disk cache,0,milvus,, milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, -milvus.querynode.disk.used_size,gauge,,,,disk used size(MB),0,milvus,, +milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, -milvus.querynode.entity.size,gauge,,,,"entities' memory size, clustered by collection and state",0,milvus,, +milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.querynode.forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, -milvus.querynode.load.index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.forward_delete_latency,gauge,,millisecond,,forward delete cost in ms,0,milvus,, +milvus.querynode.load.index_latency,gauge,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus.querynode.load.segment.latency,gauge,,,,latency of load per segment,0,milvus,, -milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.querynode.load.segment.latency,gauge,,millisecond,,latency of load per segment,0,milvus,, +milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus.querynode.process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.process_insert_or_delete_latency,gauge,,millisecond,,process insert or delete cost in ms,0,milvus,, milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, @@ -107,29 +107,29 @@ milvus.querynode.search.group.size,gauge,,,,the number of tasks of each grouped milvus.querynode.search.group.topk,gauge,,,,the topK of each grouped search task,0,milvus,, milvus.querynode.search.nq,gauge,,,,the number of queries of each search task,0,milvus,, milvus.querynode.search.topk,gauge,,,,the top of each search task,0,milvus,, -milvus.querynode.segment.access.duration,count,,,,total time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.duration,count,,millisecond,,total time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration,gauge,,millisecond,,global time cost of accessing segments,0,milvus,, milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, -milvus.querynode.segment.access.wait_cache.duration,count,,,,total time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.duration,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration,gauge,,millisecond,,global time cost of waiting for loading access,0,milvus,, milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, -milvus.querynode.segment.latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.latency_per_vector,gauge,,millisecond,,one vector's search latency per segment,0,milvus,, milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus.querynode.sq.core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.queue.latency,gauge,,,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.core_latency,gauge,,millisecond,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.queue.latency,gauge,,millisecond,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency,gauge,,millisecond,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.reduce_latency,gauge,,millisecond,,latency of reduce search or query result,0,milvus,, milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, -milvus.querynode.sq.req.latency,gauge,,,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.sq.req.latency,gauge,,millisecond,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.segment_latency,gauge,,millisecond,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency,gauge,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, -milvus.querynode.watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, +milvus.querynode.watch_dml_channel_latency,gauge,,millisecond,,latency of watch dml channel,0,milvus,, milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency,gauge,,,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.ddl_req.latency,gauge,,millisecond,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue,gauge,,millisecond,,latency of each DDL operations in queue,0,milvus,, milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, @@ -138,13 +138,13 @@ milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, cluste milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.rootcoord.produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus.rootcoord.sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, -milvus.rootcoord.timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, +milvus.rootcoord.sync_timetick_latency,gauge,,millisecond,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,, milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, -milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.storage.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, From ccea68843021a16fd2493aa0c43aaced81a38d71 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 25 Nov 2024 15:07:03 +0100 Subject: [PATCH 10/31] Remove process signature --- milvus/manifest.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/milvus/manifest.json b/milvus/manifest.json index 0f16fb61c2faa..44a7ffd9b74be 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -41,10 +41,7 @@ }, "service_checks": { "metadata_path": "assets/service_checks.json" - }, - "process_signatures": [ - "milvus" - ] + } } }, "author": { From e190fd690719d460e8e3fd987234e2a7d784de3a Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 25 Nov 2024 15:26:50 +0100 Subject: [PATCH 11/31] Add payload fixture --- milvus/tests/fixtures/docker_payload.txt | 3894 ++++++++++++++++++++++ 1 file changed, 3894 insertions(+) create mode 100644 milvus/tests/fixtures/docker_payload.txt diff --git a/milvus/tests/fixtures/docker_payload.txt b/milvus/tests/fixtures/docker_payload.txt new file mode 100644 index 0000000000000..605f8948ca370 --- /dev/null +++ b/milvus/tests/fixtures/docker_payload.txt @@ -0,0 +1,3894 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 1.3251e-05 +go_gc_duration_seconds{quantile="0.25"} 7.625e-05 +go_gc_duration_seconds{quantile="0.5"} 9.6499e-05 +go_gc_duration_seconds{quantile="0.75"} 0.000127667 +go_gc_duration_seconds{quantile="1"} 0.001003166 +go_gc_duration_seconds_sum 0.006592997 +go_gc_duration_seconds_count 33 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 360 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.21.11"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 6.5780264e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.225491696e+09 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.750476e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 1.2413451e+07 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 7.998296e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 6.5780264e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 7.7365248e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 7.3826304e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 609354 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 7.02464e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 1.51191552e+08 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.7325304762589433e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 1.3022805e+07 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 12000 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 15600 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 1.178184e+06 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 1.336272e+06 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 7.7677144e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 2.083652e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 3.997696e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 3.997696e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 1.68373544e+08 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 18 +# HELP milvus_build_info Build information of milvus +# TYPE milvus_build_info gauge +milvus_build_info{built="Tue Nov 5 07:22:45 UTC 2024",git_commit="b7c80f9",version="v2.4.15"} 1 +# HELP milvus_cgo_active_future_total Total number of active futures. +# TYPE milvus_cgo_active_future_total gauge +milvus_cgo_active_future_total{node_id="2"} 0 +# HELP milvus_cgo_cgo_duration_seconds Histogram of cgo call duration in seconds. +# TYPE milvus_cgo_cgo_duration_seconds histogram +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="9.999999999999999e-06"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1.9999999999999998e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_destroy",node_id="2"} 1.1126e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_destroy",node_id="2"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="9.999999999999999e-06"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1.9999999999999998e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_go_register_ready_callback",node_id="2"} 1.5081999999999999e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_go_register_ready_callback",node_id="2"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="9.999999999999999e-06"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1.9999999999999998e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_leak_and_get",node_id="2"} 2.5625e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_leak_and_get",node_id="2"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="4.9999999999999996e-05"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="9.999999999999999e-05"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.00025"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.0005"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.001"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.002"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.01"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="+Inf"} 5 +milvus_cgo_cgo_duration_seconds_sum{name="retrieve",node_id="2"} 0.000289917 +milvus_cgo_cgo_duration_seconds_count{name="retrieve",node_id="2"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="4.9999999999999996e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="9.999999999999999e-05"} 1 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.00025"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.0005"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.001"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.002"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.01"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="+Inf"} 2 +milvus_cgo_cgo_duration_seconds_sum{name="search",node_id="2"} 0.000204042 +milvus_cgo_cgo_duration_seconds_count{name="search",node_id="2"} 2 +# HELP milvus_cgo_cgo_queue_duration_seconds Duration of cgo call in queue. +# TYPE milvus_cgo_cgo_queue_duration_seconds histogram +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1e-08"} 0 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1.0000000000000001e-07"} 14 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="2.5000000000000004e-07"} 21 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="5.000000000000001e-07"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1e-06"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="9.999999999999999e-06"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1.9999999999999998e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="4.9999999999999996e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="9.999999999999999e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.00025"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.0005"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.001"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.002"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.01"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="+Inf"} 28 +milvus_cgo_cgo_queue_duration_seconds_sum{node_id="2"} 5.0410000000000005e-06 +milvus_cgo_cgo_queue_duration_seconds_count{node_id="2"} 28 +# HELP milvus_cgo_running_cgo_call_total Total number of running cgo calls. +# TYPE milvus_cgo_running_cgo_call_total gauge +milvus_cgo_running_cgo_call_total{node_id="2"} 0 +# HELP milvus_datacoord_collection_num number of collections +# TYPE milvus_datacoord_collection_num gauge +milvus_datacoord_collection_num 0 +# HELP milvus_datacoord_consume_datanode_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_datacoord_consume_datanode_tt_lag_ms gauge +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0",node_id="2"} 243 +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0_454172268311087164v0",node_id="2"} 243 +# HELP milvus_datacoord_datanode_num number of data nodes +# TYPE milvus_datacoord_datanode_num gauge +milvus_datacoord_datanode_num 1 +# HELP milvus_datacoord_import_tasks the import tasks grouping by type and state +# TYPE milvus_datacoord_import_tasks gauge +milvus_datacoord_import_tasks{import_state="Completed",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Completed",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Failed",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Failed",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="InProgress",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="InProgress",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Pending",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Pending",task_type="PreImportTask"} 0 +# HELP milvus_datacoord_index_node_num number of IndexNodes managed by IndexCoord +# TYPE milvus_datacoord_index_node_num gauge +milvus_datacoord_index_node_num 1 +# HELP milvus_datacoord_index_req_count number of building index requests +# TYPE milvus_datacoord_index_req_count counter +milvus_datacoord_index_req_count{status="success"} 1 +milvus_datacoord_index_req_count{status="total"} 1 +# HELP milvus_datacoord_segment_num number of segments +# TYPE milvus_datacoord_segment_num gauge +milvus_datacoord_segment_num{segment_level="L0",segment_state="Dropped"} 1 +milvus_datacoord_segment_num{segment_level="L0",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_level="L1",segment_state="Dropped"} 1 +milvus_datacoord_segment_num{segment_level="L1",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_level="L1",segment_state="Flushing"} 0 +milvus_datacoord_segment_num{segment_level="L1",segment_state="Growing"} 0 +milvus_datacoord_segment_num{segment_level="L1",segment_state="Sealed"} 0 +# HELP milvus_datacoord_task_execute_max_latency latency of task execute operation +# TYPE milvus_datacoord_task_execute_max_latency histogram +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="100"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="500"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1000"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeIndexJob"} 1540 +milvus_datacoord_task_execute_max_latency_count{status="executing",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="100"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="500"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_count{status="pending",task_type="JobTypeIndexJob"} 1 +# HELP milvus_datanode_autoflush_buffer_op_count count of auto flush buffer operations +# TYPE milvus_datanode_autoflush_buffer_op_count counter +milvus_datanode_autoflush_buffer_op_count{node_id="2",segment_level="L1",status="success"} 1 +# HELP milvus_datanode_consume_bytes_count +# TYPE milvus_datanode_consume_bytes_count counter +milvus_datanode_consume_bytes_count{msg_type="delete",node_id="2"} 231 +milvus_datanode_consume_bytes_count{msg_type="insert",node_id="2"} 191554 +# HELP milvus_datanode_encode_buffer_latency latency of encode buffer data +# TYPE milvus_datanode_encode_buffer_latency histogram +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="5"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="10"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="20"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="50"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="100"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="200"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="300"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="400"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="500"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="600"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="700"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="800"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="900"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="1000"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="1500"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="3000"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="10000"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="60000"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="+Inf"} 1 +milvus_datanode_encode_buffer_latency_sum{node_id="2",segment_level="L0"} 4 +milvus_datanode_encode_buffer_latency_count{node_id="2",segment_level="L0"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="5"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="10"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="20"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="50"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="100"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="200"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="300"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="400"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="500"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="600"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="700"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="800"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="900"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="1000"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="1500"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="3000"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="10000"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="60000"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="+Inf"} 2 +milvus_datanode_encode_buffer_latency_sum{node_id="2",segment_level="L1"} 16 +milvus_datanode_encode_buffer_latency_count{node_id="2",segment_level="L1"} 2 +# HELP milvus_datanode_flowgraph_num number of flowgraphs +# TYPE milvus_datanode_flowgraph_num gauge +milvus_datanode_flowgraph_num{node_id="2"} 0 +# HELP milvus_datanode_flush_buffer_op_count count of flush buffer operations +# TYPE milvus_datanode_flush_buffer_op_count counter +milvus_datanode_flush_buffer_op_count{node_id="2",segment_level="L0",status="success"} 1 +milvus_datanode_flush_buffer_op_count{node_id="2",segment_level="L1",status="success"} 2 +# HELP milvus_datanode_flush_req_count count of flush request +# TYPE milvus_datanode_flush_req_count counter +milvus_datanode_flush_req_count{node_id="2",status="success"} 1 +milvus_datanode_flush_req_count{node_id="2",status="total"} 1 +# HELP milvus_datanode_flushed_data_rows num of rows flushed to storage +# TYPE milvus_datanode_flushed_data_rows counter +milvus_datanode_flushed_data_rows{data_source="streaming",node_id="2"} 3001 +# HELP milvus_datanode_flushed_data_size byte size of data flushed to storage +# TYPE milvus_datanode_flushed_data_size counter +milvus_datanode_flushed_data_size{data_source="streaming",node_id="2",segment_level="L0"} 616 +milvus_datanode_flushed_data_size{data_source="streaming",node_id="2",segment_level="L1"} 226975 +# HELP milvus_datanode_msg_rows_count count of rows consumed from msgStream +# TYPE milvus_datanode_msg_rows_count counter +milvus_datanode_msg_rows_count{msg_type="delete",node_id="2"} 2 +milvus_datanode_msg_rows_count{msg_type="insert",node_id="2"} 3001 +# HELP milvus_datanode_save_latency latency of saving flush data to storage +# TYPE milvus_datanode_save_latency histogram +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="10"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="100"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="200"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="400"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="1000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="10000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="+Inf"} 1 +milvus_datanode_save_latency_sum{msg_type="L0",node_id="2"} 3 +milvus_datanode_save_latency_count{msg_type="L0",node_id="2"} 1 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="10"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="100"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="200"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="400"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="1000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="10000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="+Inf"} 2 +milvus_datanode_save_latency_sum{msg_type="L1",node_id="2"} 8 +milvus_datanode_save_latency_count{msg_type="L1",node_id="2"} 2 +# HELP milvus_flushed_segment_file_num the num of files for flushed segment +# TYPE milvus_flushed_segment_file_num histogram +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="5"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="10"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="20"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="50"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="100"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="200"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="300"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="400"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="600"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="700"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="800"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="900"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="3000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="10000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="delete_file"} 0 +milvus_flushed_segment_file_num_count{segment_file_type="delete_file"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="5"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="10"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="20"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="50"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="100"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="200"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="300"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="400"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="600"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="700"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="800"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="900"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="3000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="10000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="index_file"} 1 +milvus_flushed_segment_file_num_count{segment_file_type="index_file"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="5"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="10"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="20"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="50"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="100"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="200"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="300"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="400"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="600"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="700"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="800"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="900"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="3000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="10000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="insert_file"} 5 +milvus_flushed_segment_file_num_count{segment_file_type="insert_file"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="5"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="10"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="20"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="50"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="100"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="200"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="300"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="400"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="600"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="700"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="800"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="900"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1500"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="3000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="10000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="stat_file"} 2 +milvus_flushed_segment_file_num_count{segment_file_type="stat_file"} 1 +# HELP milvus_indexnode_build_index_latency latency of build index for segment +# TYPE milvus_indexnode_build_index_latency histogram +milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.001"} 0 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="10"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="20"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="50"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="100"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="250"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="500"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="1000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="3600"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="5000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="10000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_indexnode_build_index_latency_sum{node_id="2"} 0.026148791 +milvus_indexnode_build_index_latency_count{node_id="2"} 1 +# HELP milvus_indexnode_encode_index_latency latency of encoding the index file +# TYPE milvus_indexnode_encode_index_latency histogram +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.001"} 0 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="10"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="20"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="50"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="100"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="250"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="500"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="1000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="3600"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="5000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="10000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_indexnode_encode_index_latency_sum{node_id="2"} 0.00681125 +milvus_indexnode_encode_index_latency_count{node_id="2"} 1 +# HELP milvus_indexnode_index_task_count number of tasks that index node received +# TYPE milvus_indexnode_index_task_count counter +milvus_indexnode_index_task_count{node_id="2",status="success"} 1 +# HELP milvus_indexnode_index_task_latency_in_queue latency of index task in queue +# TYPE milvus_indexnode_index_task_latency_in_queue histogram +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="5"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="10"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="20"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="50"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="100"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="200"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="300"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="400"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="500"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="600"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="700"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="800"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="900"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="1000"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="1500"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="3000"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="10000"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="60000"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="+Inf"} 1 +milvus_indexnode_index_task_latency_in_queue_sum{node_id="2"} 0 +milvus_indexnode_index_task_latency_in_queue_count{node_id="2"} 1 +# HELP milvus_indexnode_knowhere_build_index_latency latency of building the index by knowhere +# TYPE milvus_indexnode_knowhere_build_index_latency histogram +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.001"} 0 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="10"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="20"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="50"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="100"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="250"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="500"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="1000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="3600"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="5000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="10000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_indexnode_knowhere_build_index_latency_sum{node_id="2"} 0.019188333 +milvus_indexnode_knowhere_build_index_latency_count{node_id="2"} 1 +# HELP milvus_indexnode_save_index_latency latency of saving the index file +# TYPE milvus_indexnode_save_index_latency histogram +milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.001"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="10"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="20"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="50"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="100"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="250"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="500"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="1000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="3600"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="5000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="10000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_indexnode_save_index_latency_sum{node_id="2"} 4.6958e-05 +milvus_indexnode_save_index_latency_count{node_id="2"} 1 +# HELP milvus_meta_kv_size kv size stats +# TYPE milvus_meta_kv_size histogram +milvus_meta_kv_size_bucket{meta_op_type="get",le="5"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="10"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="20"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="50"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="100"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="200"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="300"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="400"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="500"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="600"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="700"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="800"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="900"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="1000"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="1500"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="3000"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="10000"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="60000"} 77 +milvus_meta_kv_size_bucket{meta_op_type="get",le="+Inf"} 77 +milvus_meta_kv_size_sum{meta_op_type="get"} -20 +milvus_meta_kv_size_count{meta_op_type="get"} 77 +milvus_meta_kv_size_bucket{meta_op_type="put",le="5"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="10"} 273 +milvus_meta_kv_size_bucket{meta_op_type="put",le="20"} 274 +milvus_meta_kv_size_bucket{meta_op_type="put",le="50"} 282 +milvus_meta_kv_size_bucket{meta_op_type="put",le="100"} 290 +milvus_meta_kv_size_bucket{meta_op_type="put",le="200"} 293 +milvus_meta_kv_size_bucket{meta_op_type="put",le="300"} 293 +milvus_meta_kv_size_bucket{meta_op_type="put",le="400"} 299 +milvus_meta_kv_size_bucket{meta_op_type="put",le="500"} 302 +milvus_meta_kv_size_bucket{meta_op_type="put",le="600"} 302 +milvus_meta_kv_size_bucket{meta_op_type="put",le="700"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="800"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="900"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1000"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1500"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="3000"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="10000"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="60000"} 305 +milvus_meta_kv_size_bucket{meta_op_type="put",le="+Inf"} 305 +milvus_meta_kv_size_sum{meta_op_type="put"} 8855 +milvus_meta_kv_size_count{meta_op_type="put"} 305 +# HELP milvus_meta_op_count count of meta operation +# TYPE milvus_meta_op_count counter +milvus_meta_op_count{meta_op_type="get",status="success"} 52 +milvus_meta_op_count{meta_op_type="get",status="total"} 52 +milvus_meta_op_count{meta_op_type="put",status="success"} 280 +milvus_meta_op_count{meta_op_type="put",status="total"} 280 +milvus_meta_op_count{meta_op_type="remove",status="success"} 6 +milvus_meta_op_count{meta_op_type="remove",status="total"} 6 +milvus_meta_op_count{meta_op_type="txn",status="success"} 25 +milvus_meta_op_count{meta_op_type="txn",status="total"} 25 +# HELP milvus_meta_request_latency request latency on the client side +# TYPE milvus_meta_request_latency histogram +milvus_meta_request_latency_bucket{meta_op_type="get",le="5"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="10"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="20"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="50"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="100"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="200"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="300"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="400"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="500"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="600"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="700"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="800"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="900"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="1000"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="1500"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="3000"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="10000"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="60000"} 52 +milvus_meta_request_latency_bucket{meta_op_type="get",le="+Inf"} 52 +milvus_meta_request_latency_sum{meta_op_type="get"} 0 +milvus_meta_request_latency_count{meta_op_type="get"} 52 +milvus_meta_request_latency_bucket{meta_op_type="put",le="5"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="10"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="20"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="50"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="100"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="200"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="300"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="400"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="500"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="600"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="700"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="800"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="900"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1000"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1500"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="3000"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="10000"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="60000"} 280 +milvus_meta_request_latency_bucket{meta_op_type="put",le="+Inf"} 280 +milvus_meta_request_latency_sum{meta_op_type="put"} 16 +milvus_meta_request_latency_count{meta_op_type="put"} 280 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="5"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="10"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="20"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="50"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="100"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="200"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="300"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="400"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="500"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="600"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="700"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="800"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="900"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1000"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1500"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="3000"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="10000"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="60000"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="+Inf"} 6 +milvus_meta_request_latency_sum{meta_op_type="remove"} 0 +milvus_meta_request_latency_count{meta_op_type="remove"} 6 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="5"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="10"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="20"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="50"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="100"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="200"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="300"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="400"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="500"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="600"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="700"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="800"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="900"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1000"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1500"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="3000"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="10000"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="60000"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="+Inf"} 25 +milvus_meta_request_latency_sum{meta_op_type="txn"} 1 +milvus_meta_request_latency_count{meta_op_type="txn"} 25 +# HELP milvus_msg_queue_consumer_num number of consumers +# TYPE milvus_msg_queue_consumer_num gauge +milvus_msg_queue_consumer_num{node_id="2",role_name="standalone"} 0 +# HELP milvus_msgstream_op_count count of stream message operation +# TYPE milvus_msgstream_op_count counter +milvus_msgstream_op_count{message_op_type="create_consumer",status="success"} 2 +milvus_msgstream_op_count{message_op_type="create_consumer",status="total"} 2 +milvus_msgstream_op_count{message_op_type="create_producer",status="success"} 18 +milvus_msgstream_op_count{message_op_type="create_producer",status="total"} 18 +milvus_msgstream_op_count{message_op_type="produce",status="success"} 59 +milvus_msgstream_op_count{message_op_type="produce",status="total"} 59 +# HELP milvus_msgstream_request_latency request latency on the client side +# TYPE milvus_msgstream_request_latency histogram +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="5"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="10"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="20"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="50"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="100"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="200"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="300"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="400"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="500"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="600"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="700"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="800"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="900"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1000"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1500"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="3000"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="10000"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="60000"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="+Inf"} 2 +milvus_msgstream_request_latency_sum{message_op_type="create_consumer"} 0 +milvus_msgstream_request_latency_count{message_op_type="create_consumer"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="5"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="10"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="20"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="50"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="100"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="200"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="300"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="400"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="500"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="600"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="700"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="800"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="900"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1000"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1500"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="3000"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="10000"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="60000"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="+Inf"} 18 +milvus_msgstream_request_latency_sum{message_op_type="create_producer"} 0 +milvus_msgstream_request_latency_count{message_op_type="create_producer"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="5"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="10"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="20"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="50"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="100"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="200"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="300"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="400"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="500"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="600"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="700"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="800"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="900"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1000"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1500"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="3000"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="10000"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="60000"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="+Inf"} 59 +milvus_msgstream_request_latency_sum{message_op_type="produce"} 0 +milvus_msgstream_request_latency_count{message_op_type="produce"} 59 +# HELP milvus_num_node number of nodes and coordinates +# TYPE milvus_num_node gauge +milvus_num_node{node_id="2",role_name="datacoord"} 1 +milvus_num_node{node_id="2",role_name="datanode"} 1 +milvus_num_node{node_id="2",role_name="indexnode"} 1 +milvus_num_node{node_id="2",role_name="proxy"} 1 +milvus_num_node{node_id="2",role_name="querycoord"} 1 +milvus_num_node{node_id="2",role_name="querynode"} 1 +milvus_num_node{node_id="2",role_name="rootcoord"} 1 +# HELP milvus_proxy_apply_pk_latency latency that apply primary key +# TYPE milvus_proxy_apply_pk_latency histogram +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="5"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="10"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="20"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="50"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="100"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="200"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="300"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="400"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="500"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="600"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="700"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="800"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="900"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="1000"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="1500"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="3000"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="10000"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="60000"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="2",le="+Inf"} 2 +milvus_proxy_apply_pk_latency_sum{node_id="2"} 0 +milvus_proxy_apply_pk_latency_count{node_id="2"} 2 +# HELP milvus_proxy_apply_timestamp_latency latency that proxy apply timestamp +# TYPE milvus_proxy_apply_timestamp_latency histogram +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="5"} 8041 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="10"} 8042 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="20"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="50"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="100"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="200"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="300"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="400"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="500"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="600"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="700"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="800"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="900"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="1000"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="1500"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="3000"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="10000"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="60000"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="+Inf"} 8043 +milvus_proxy_apply_timestamp_latency_sum{node_id="2"} 42 +milvus_proxy_apply_timestamp_latency_count{node_id="2"} 8043 +# HELP milvus_proxy_assign_segmentID_latency latency that proxy get segmentID from dataCoord +# TYPE milvus_proxy_assign_segmentID_latency histogram +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="5"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="10"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="20"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="50"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="100"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="200"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="300"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="400"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="500"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="600"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="700"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="800"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="900"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="1000"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="1500"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="3000"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="10000"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="60000"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="+Inf"} 2 +milvus_proxy_assign_segmentID_latency_sum{node_id="2"} 1 +milvus_proxy_assign_segmentID_latency_count{node_id="2"} 2 +# HELP milvus_proxy_cache_hit_count count of cache hits/miss +# TYPE milvus_proxy_cache_hit_count counter +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="hit",node_id="2"} 54 +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="miss",node_id="2"} 3 +milvus_proxy_cache_hit_count{cache_name="GetCollectionInfo",cache_state="hit",node_id="2"} 15 +milvus_proxy_cache_hit_count{cache_name="GetCollectionSchema",cache_state="hit",node_id="2"} 25 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="hit",node_id="2"} 6 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="miss",node_id="2"} 1 +# HELP milvus_proxy_cache_update_latency latency that proxy update cache when cache miss +# TYPE milvus_proxy_cache_update_latency histogram +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="5"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="10"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="20"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="50"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="100"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="200"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="300"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="400"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="500"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="600"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="700"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="800"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="900"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="1000"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="1500"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="3000"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="10000"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="60000"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="+Inf"} 2 +milvus_proxy_cache_update_latency_sum{cache_name="GetCollectionID",node_id="2"} 1 +milvus_proxy_cache_update_latency_count{cache_name="GetCollectionID",node_id="2"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="5"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="10"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="20"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="50"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="100"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="200"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="300"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="400"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="500"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="600"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="700"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="800"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="900"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="1000"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="1500"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="3000"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="10000"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="60000"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="+Inf"} 1 +milvus_proxy_cache_update_latency_sum{cache_name="GetShards",node_id="2"} 0 +milvus_proxy_cache_update_latency_count{cache_name="GetShards",node_id="2"} 1 +# HELP milvus_proxy_delete_vectors_count counter of vectors successfully deleted +# TYPE milvus_proxy_delete_vectors_count counter +milvus_proxy_delete_vectors_count{db_name="default",node_id="2"} 2 +# HELP milvus_proxy_msgstream_obj_num number of MsgStream objects per physical channel +# TYPE milvus_proxy_msgstream_obj_num gauge +milvus_proxy_msgstream_obj_num{channel_name="by-dev-rootcoord-dml_0",node_id="2"} 0 +# HELP milvus_proxy_mutation_send_latency latency that proxy send insert request to MsgStream +# TYPE milvus_proxy_mutation_send_latency histogram +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="100"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="500"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="5000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="10000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="20000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="50000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="100000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="250000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="500000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="3.6e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="5e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1e+07"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="+Inf"} 2 +milvus_proxy_mutation_send_latency_sum{msg_type="insert",node_id="2"} 0 +milvus_proxy_mutation_send_latency_count{msg_type="insert",node_id="2"} 2 +# HELP milvus_proxy_rate_limit_req_count count of operation executed +# TYPE milvus_proxy_rate_limit_req_count counter +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="2",status="success"} 27 +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="2",status="total"} 27 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="2",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="2",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="2",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="2",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="2",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="2",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="2",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="2",status="total"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="2",status="success"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="2",status="total"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="2",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="2",status="total"} 2 +# HELP milvus_proxy_report_value report value about the request +# TYPE milvus_proxy_report_value counter +milvus_proxy_report_value{db_name="default",msg_type="delete",node_id="2",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="insert",node_id="2",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="query",node_id="2",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="search",node_id="2",username=""} 0 +# HELP milvus_proxy_req_count count of operation executed +# TYPE milvus_proxy_req_count counter +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="2",status="success"} 1 +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="2",status="total"} 1 +milvus_proxy_req_count{collection_name="hello_milvus",db_name="default",function_name="DropCollection",node_id="2",status="success"} 1 +# HELP milvus_proxy_req_in_queue_latency latency which request waits in the queue +# TYPE milvus_proxy_req_in_queue_latency histogram +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateCollection",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateCollection",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateIndex",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateIndex",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Delete",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Delete",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="5"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="10"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="20"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="50"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="100"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="200"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="300"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="400"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="500"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="600"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="700"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="800"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="900"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="1000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="1500"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="3000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="10000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="60000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="+Inf"} 4 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeCollection",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeCollection",node_id="2"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="5"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="10"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="20"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="50"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="100"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="200"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="300"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="400"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="500"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="600"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="700"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="800"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="900"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="1000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="1500"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="3000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="10000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="60000"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="+Inf"} 4 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeIndex",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeIndex",node_id="2"} 4 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="DropCollection",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DropCollection",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Flush",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Flush",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="GetCollectionStatistics",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="GetCollectionStatistics",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="5"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="10"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="20"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="50"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="100"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="200"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="300"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="400"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="500"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="600"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="700"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="800"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="900"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="1000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="1500"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="3000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="10000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="60000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Insert",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Insert",node_id="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="5"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="10"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="20"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="50"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="100"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="200"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="300"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="400"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="600"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="700"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="800"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="900"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="LoadCollection",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="LoadCollection",node_id="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="5"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="10"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="20"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="50"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="100"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="200"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="300"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="400"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="500"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="600"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="700"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="800"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="900"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="1000"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="1500"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="3000"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="10000"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="60000"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="+Inf"} 5 +milvus_proxy_req_in_queue_latency_sum{function_name="Retrieve",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Retrieve",node_id="2"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="5"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="10"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="20"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="50"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="100"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="200"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="300"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="400"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="500"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="600"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="700"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="800"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="900"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="1000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="1500"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="3000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="10000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="60000"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Search",node_id="2"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Search",node_id="2"} 2 +# HELP milvus_proxy_req_latency latency of each request +# TYPE milvus_proxy_req_latency histogram +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="5"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateCollection",node_id="2"} 7 +milvus_proxy_req_latency_count{function_name="CreateCollection",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="5"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateIndex",node_id="2"} 2 +milvus_proxy_req_latency_count{function_name="CreateIndex",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="5"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="10"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="20"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="50"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="100"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="200"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="300"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="400"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="500"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="600"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="700"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="800"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="900"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="1000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="1500"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="3000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="10000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="60000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="+Inf"} 4 +milvus_proxy_req_latency_sum{function_name="DescribeCollection",node_id="2"} 0 +milvus_proxy_req_latency_count{function_name="DescribeCollection",node_id="2"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="5"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="10"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="20"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="50"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="100"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="200"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="300"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="400"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="500"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="600"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="700"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="800"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="900"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="1000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="1500"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="3000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="10000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="60000"} 4 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="+Inf"} 4 +milvus_proxy_req_latency_sum{function_name="DescribeIndex",node_id="2"} 0 +milvus_proxy_req_latency_count{function_name="DescribeIndex",node_id="2"} 4 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="5"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="DropCollection",node_id="2"} 1 +milvus_proxy_req_latency_count{function_name="DropCollection",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="5"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="Flush",node_id="2"} 1 +milvus_proxy_req_latency_count{function_name="Flush",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="5"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="GetCollectionStatistics",node_id="2"} 0 +milvus_proxy_req_latency_count{function_name="GetCollectionStatistics",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="5"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="10"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="20"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="50"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="100"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="200"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="300"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="400"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="500"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="600"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="700"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="800"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="900"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="1000"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="1500"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="3000"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="10000"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="60000"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="+Inf"} 6 +milvus_proxy_req_latency_sum{function_name="GetLoadingProgress",node_id="2"} 0 +milvus_proxy_req_latency_count{function_name="GetLoadingProgress",node_id="2"} 6 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="5"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="10"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="20"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="50"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="100"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="200"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="300"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="400"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="500"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="600"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="700"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="800"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="900"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="1000"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="1500"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="3000"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="10000"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="60000"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="LoadCollection",node_id="2"} 6 +milvus_proxy_req_latency_count{function_name="LoadCollection",node_id="2"} 1 +# HELP milvus_proxy_send_bytes_count count of bytes sent back to sdk +# TYPE milvus_proxy_send_bytes_count counter +milvus_proxy_send_bytes_count{node_id="2"} 70813 +# HELP milvus_proxy_sq_decode_result_latency latency that proxy decodes the search result +# TYPE milvus_proxy_sq_decode_result_latency histogram +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="5"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="10"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="20"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="50"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="100"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="200"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="300"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_decode_result_latency_sum{node_id="2",query_type="query"} 0 +milvus_proxy_sq_decode_result_latency_count{node_id="2",query_type="query"} 5 +# HELP milvus_proxy_sq_reduce_result_latency latency that proxy reduces search result +# TYPE milvus_proxy_sq_reduce_result_latency histogram +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="5"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="10"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="20"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="50"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="100"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="200"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="300"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_reduce_result_latency_sum{node_id="2",query_type="query"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="2",query_type="query"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="5"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="10"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="20"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="50"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="100"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="200"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="300"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="400"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="500"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="600"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="700"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="800"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="900"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="1000"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="1500"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="3000"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="10000"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="60000"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_reduce_result_latency_sum{node_id="2",query_type="search"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="2",query_type="search"} 2 +# HELP milvus_proxy_sq_wait_result_latency latency that proxy waits for the result +# TYPE milvus_proxy_sq_wait_result_latency histogram +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="5"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="10"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="20"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="50"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="100"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="200"} 4 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="300"} 4 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_wait_result_latency_sum{node_id="2",query_type="query"} 1132 +milvus_proxy_sq_wait_result_latency_count{node_id="2",query_type="query"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="5"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="10"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="20"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="50"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="100"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="200"} 1 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="300"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="400"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="500"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="600"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="700"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="800"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="900"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="1000"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="1500"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="3000"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="10000"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="60000"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_wait_result_latency_sum{node_id="2",query_type="search"} 470 +milvus_proxy_sq_wait_result_latency_count{node_id="2",query_type="search"} 2 +# HELP milvus_proxy_sync_segment_request_length the length of SegmentIDRequests when assigning segments for insert +# TYPE milvus_proxy_sync_segment_request_length histogram +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="5"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="10"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="20"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="50"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="100"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="200"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="300"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="400"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="500"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="600"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="700"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="800"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="900"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="1000"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="1500"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="3000"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="10000"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="60000"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="+Inf"} 2 +milvus_proxy_sync_segment_request_length_sum{node_id="2"} 2 +milvus_proxy_sync_segment_request_length_count{node_id="2"} 2 +# HELP milvus_proxy_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_proxy_tt_lag_ms gauge +milvus_proxy_tt_lag_ms{channel_name="default",node_id="2"} 141 +# HELP milvus_querycoord_collection_num number of collections +# TYPE milvus_querycoord_collection_num gauge +milvus_querycoord_collection_num 0 +# HELP milvus_querycoord_load_latency latency of load the entire collection +# TYPE milvus_querycoord_load_latency histogram +milvus_querycoord_load_latency_bucket{le="0"} 0 +milvus_querycoord_load_latency_bucket{le="500"} 0 +milvus_querycoord_load_latency_bucket{le="1000"} 2 +milvus_querycoord_load_latency_bucket{le="2000"} 2 +milvus_querycoord_load_latency_bucket{le="5000"} 2 +milvus_querycoord_load_latency_bucket{le="10000"} 2 +milvus_querycoord_load_latency_bucket{le="20000"} 2 +milvus_querycoord_load_latency_bucket{le="50000"} 2 +milvus_querycoord_load_latency_bucket{le="60000"} 2 +milvus_querycoord_load_latency_bucket{le="300000"} 2 +milvus_querycoord_load_latency_bucket{le="600000"} 2 +milvus_querycoord_load_latency_bucket{le="1.8e+06"} 2 +milvus_querycoord_load_latency_bucket{le="+Inf"} 2 +milvus_querycoord_load_latency_sum 1805 +milvus_querycoord_load_latency_count 2 +# HELP milvus_querycoord_load_req_count count of load request +# TYPE milvus_querycoord_load_req_count counter +milvus_querycoord_load_req_count{status="success"} 1 +milvus_querycoord_load_req_count{status="total"} 1 +# HELP milvus_querycoord_partition_num number of partitions +# TYPE milvus_querycoord_partition_num gauge +milvus_querycoord_partition_num 0 +# HELP milvus_querycoord_querynode_num number of QueryNodes managered by QueryCoord +# TYPE milvus_querycoord_querynode_num gauge +milvus_querycoord_querynode_num 1 +# HELP milvus_querycoord_release_latency latency of release request +# TYPE milvus_querycoord_release_latency histogram +milvus_querycoord_release_latency_bucket{le="0"} 0 +milvus_querycoord_release_latency_bucket{le="5"} 0 +milvus_querycoord_release_latency_bucket{le="10"} 0 +milvus_querycoord_release_latency_bucket{le="20"} 0 +milvus_querycoord_release_latency_bucket{le="40"} 0 +milvus_querycoord_release_latency_bucket{le="100"} 0 +milvus_querycoord_release_latency_bucket{le="200"} 0 +milvus_querycoord_release_latency_bucket{le="400"} 0 +milvus_querycoord_release_latency_bucket{le="1000"} 1 +milvus_querycoord_release_latency_bucket{le="10000"} 1 +milvus_querycoord_release_latency_bucket{le="+Inf"} 1 +milvus_querycoord_release_latency_sum 605 +milvus_querycoord_release_latency_count 1 +# HELP milvus_querycoord_release_req_count count of release request +# TYPE milvus_querycoord_release_req_count counter +milvus_querycoord_release_req_count{status="success"} 1 +milvus_querycoord_release_req_count{status="total"} 1 +# HELP milvus_querycoord_task_num the number of tasks in QueryCoord's scheduler +# TYPE milvus_querycoord_task_num gauge +milvus_querycoord_task_num{querycoord_task_type="channel_grow"} 0 +milvus_querycoord_task_num{querycoord_task_type="channel_move"} 0 +milvus_querycoord_task_num{querycoord_task_type="channel_reduce"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_grow"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_move"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_reduce"} 0 +# HELP milvus_querynode_apply_bf_latency apply bf cost in ms +# TYPE milvus_querynode_apply_bf_latency histogram +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="5"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="10"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="20"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="50"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="100"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="200"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="300"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="400"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="500"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="600"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="700"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="800"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="900"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="1000"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="1500"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="3000"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="10000"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="60000"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="+Inf"} 1 +milvus_querynode_apply_bf_latency_sum{function_name="ProcessDelete",node_id="2"} 0 +milvus_querynode_apply_bf_latency_count{function_name="ProcessDelete",node_id="2"} 1 +# HELP milvus_querynode_collection_num number of collections loaded +# TYPE milvus_querynode_collection_num gauge +milvus_querynode_collection_num{node_id="2"} 0 +# HELP milvus_querynode_consume_bytes_counter +# TYPE milvus_querynode_consume_bytes_counter counter +milvus_querynode_consume_bytes_counter{msg_type="insert",node_id="2"} 231 +# HELP milvus_querynode_disk_cache_evict_bytes number of bytes evicted from disk cache +# TYPE milvus_querynode_disk_cache_evict_bytes counter +milvus_querynode_disk_cache_evict_bytes{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_evict_duration total time cost of evicting segments from disk cache +# TYPE milvus_querynode_disk_cache_evict_duration counter +milvus_querynode_disk_cache_evict_duration{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_evict_global_duration global duration of evicting segments from disk cache +# TYPE milvus_querynode_disk_cache_evict_global_duration histogram +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="100"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="500"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="5000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="10000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="20000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="50000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="100000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="250000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="500000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="3.6e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="5e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1e+07"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="+Inf"} 0 +milvus_querynode_disk_cache_evict_global_duration_sum{node_id="2"} 0 +milvus_querynode_disk_cache_evict_global_duration_count{node_id="2"} 0 +# HELP milvus_querynode_disk_cache_evict_total number of segments evicted from disk cache +# TYPE milvus_querynode_disk_cache_evict_total counter +milvus_querynode_disk_cache_evict_total{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_bytes number of bytes loaded from disk cache +# TYPE milvus_querynode_disk_cache_load_bytes counter +milvus_querynode_disk_cache_load_bytes{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_duration total time cost of loading segments from disk cache +# TYPE milvus_querynode_disk_cache_load_duration counter +milvus_querynode_disk_cache_load_duration{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_global_duration global duration of loading segments from disk cache +# TYPE milvus_querynode_disk_cache_load_global_duration histogram +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="100"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="500"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="5000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="10000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="20000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="50000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="100000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="250000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="500000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="3.6e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="5e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1e+07"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="+Inf"} 0 +milvus_querynode_disk_cache_load_global_duration_sum{node_id="2"} 0 +milvus_querynode_disk_cache_load_global_duration_count{node_id="2"} 0 +# HELP milvus_querynode_disk_cache_load_total number of segments loaded from disk cache +# TYPE milvus_querynode_disk_cache_load_total counter +milvus_querynode_disk_cache_load_total{db_name="default",node_id="2",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_used_size disk used size(MB) +# TYPE milvus_querynode_disk_used_size gauge +milvus_querynode_disk_used_size{node_id="2"} 0.2461862564086914 +# HELP milvus_querynode_dml_vchannel_num number of dmlChannels watched +# TYPE milvus_querynode_dml_vchannel_num gauge +milvus_querynode_dml_vchannel_num{node_id="2"} 0 +# HELP milvus_querynode_execute_bytes_counter +# TYPE milvus_querynode_execute_bytes_counter counter +milvus_querynode_execute_bytes_counter{msg_type="query",node_id="2"} 813 +milvus_querynode_execute_bytes_counter{msg_type="search",node_id="2"} 553 +# HELP milvus_querynode_flowgraph_num number of flowgraphs +# TYPE milvus_querynode_flowgraph_num gauge +milvus_querynode_flowgraph_num{node_id="2"} 0 +# HELP milvus_querynode_forward_delete_latency forward delete cost in ms +# TYPE milvus_querynode_forward_delete_latency histogram +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="5"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="10"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="20"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="50"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="100"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="200"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="300"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="400"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="500"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="600"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="700"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="800"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="900"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="1000"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="1500"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="3000"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="10000"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="60000"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="+Inf"} 1 +milvus_querynode_forward_delete_latency_sum{function_name="ProcessDelete",node_id="2"} 0 +milvus_querynode_forward_delete_latency_count{function_name="ProcessDelete",node_id="2"} 1 +# HELP milvus_querynode_load_index_latency latency of load per segment's index, in milliseconds +# TYPE milvus_querynode_load_index_latency histogram +milvus_querynode_load_index_latency_bucket{node_id="2",le="1"} 0 +milvus_querynode_load_index_latency_bucket{node_id="2",le="100"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="500"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="1000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="5000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="10000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="20000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="50000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="100000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="250000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="500000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="1e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="3.6e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="5e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="1e+07"} 1 +milvus_querynode_load_index_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_querynode_load_index_latency_sum{node_id="2"} 3 +milvus_querynode_load_index_latency_count{node_id="2"} 1 +# HELP milvus_querynode_load_segment_concurrency number of concurrent loading segments in QueryNode +# TYPE milvus_querynode_load_segment_concurrency gauge +milvus_querynode_load_segment_concurrency{load_type="LoadSegment",node_id="2"} 0 +# HELP milvus_querynode_load_segment_latency latency of load per segment +# TYPE milvus_querynode_load_segment_latency histogram +milvus_querynode_load_segment_latency_bucket{node_id="2",le="1"} 0 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="100"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="500"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="1000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="5000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="10000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="20000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="50000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="100000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="250000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="500000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="1e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="3.6e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="5e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="1e+07"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_querynode_load_segment_latency_sum{node_id="2"} 22 +milvus_querynode_load_segment_latency_count{node_id="2"} 1 +# HELP milvus_querynode_partition_num number of partitions loaded +# TYPE milvus_querynode_partition_num gauge +milvus_querynode_partition_num{node_id="2"} 0 +# HELP milvus_querynode_process_insert_or_delete_latency process insert or delete cost in ms +# TYPE milvus_querynode_process_insert_or_delete_latency histogram +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="5"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="10"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="20"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="50"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="100"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="200"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="300"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="400"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="500"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="600"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="700"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="800"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="900"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="1000"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="1500"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="3000"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="10000"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="60000"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="+Inf"} 1 +milvus_querynode_process_insert_or_delete_latency_sum{msg_type="delete",node_id="2"} 1 +milvus_querynode_process_insert_or_delete_latency_count{msg_type="delete",node_id="2"} 1 +# HELP milvus_querynode_read_task_concurrency number of concurrent executing read tasks in QueryNode +# TYPE milvus_querynode_read_task_concurrency gauge +milvus_querynode_read_task_concurrency{node_id="2"} 0 +# HELP milvus_querynode_read_task_ready_len number of ready read tasks in readyQueue +# TYPE milvus_querynode_read_task_ready_len gauge +milvus_querynode_read_task_ready_len{node_id="2"} 0 +# HELP milvus_querynode_read_task_unsolved_len number of unsolved read tasks in unsolvedQueue +# TYPE milvus_querynode_read_task_unsolved_len gauge +milvus_querynode_read_task_unsolved_len{node_id="2"} 0 +# HELP milvus_querynode_search_group_nq the number of queries of each grouped search task +# TYPE milvus_querynode_search_group_nq histogram +milvus_querynode_search_group_nq_bucket{node_id="2",le="5"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="10"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="20"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="50"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="100"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="200"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="300"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="400"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="500"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="600"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="700"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="800"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="900"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="1000"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="1500"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="3000"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="10000"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="60000"} 2 +milvus_querynode_search_group_nq_bucket{node_id="2",le="+Inf"} 2 +milvus_querynode_search_group_nq_sum{node_id="2"} 4 +milvus_querynode_search_group_nq_count{node_id="2"} 2 +# HELP milvus_querynode_search_group_size the number of tasks of each grouped search task +# TYPE milvus_querynode_search_group_size histogram +milvus_querynode_search_group_size_bucket{node_id="2",le="5"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="10"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="20"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="50"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="100"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="200"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="300"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="400"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="500"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="600"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="700"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="800"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="900"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="1000"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="1500"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="3000"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="10000"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="60000"} 2 +milvus_querynode_search_group_size_bucket{node_id="2",le="+Inf"} 2 +milvus_querynode_search_group_size_sum{node_id="2"} 2 +milvus_querynode_search_group_size_count{node_id="2"} 2 +# HELP milvus_querynode_search_group_topk the topK of each grouped search task +# TYPE milvus_querynode_search_group_topk histogram +milvus_querynode_search_group_topk_bucket{node_id="2",le="5"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="10"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="20"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="50"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="100"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="200"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="300"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="400"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="500"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="600"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="700"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="800"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="900"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="1000"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="1500"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="3000"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="10000"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="60000"} 2 +milvus_querynode_search_group_topk_bucket{node_id="2",le="+Inf"} 2 +milvus_querynode_search_group_topk_sum{node_id="2"} 6 +milvus_querynode_search_group_topk_count{node_id="2"} 2 +# HELP milvus_querynode_search_nq the number of queries of each search task +# TYPE milvus_querynode_search_nq histogram +milvus_querynode_search_nq_bucket{node_id="2",le="5"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="10"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="20"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="50"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="100"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="200"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="300"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="400"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="500"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="600"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="700"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="800"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="900"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="1000"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="1500"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="3000"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="10000"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="60000"} 2 +milvus_querynode_search_nq_bucket{node_id="2",le="+Inf"} 2 +milvus_querynode_search_nq_sum{node_id="2"} 4 +milvus_querynode_search_nq_count{node_id="2"} 2 +# HELP milvus_querynode_search_topk the top of each search task +# TYPE milvus_querynode_search_topk histogram +milvus_querynode_search_topk_bucket{node_id="2",le="5"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="10"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="20"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="50"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="100"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="200"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="300"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="400"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="500"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="600"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="700"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="800"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="900"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="1000"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="1500"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="3000"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="10000"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="60000"} 2 +milvus_querynode_search_topk_bucket{node_id="2",le="+Inf"} 2 +milvus_querynode_search_topk_sum{node_id="2"} 6 +milvus_querynode_search_topk_count{node_id="2"} 2 +# HELP milvus_querynode_segment_access_duration total time cost of accessing segments +# TYPE milvus_querynode_segment_access_duration counter +milvus_querynode_segment_access_duration{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 3.8771669999999996 +milvus_querynode_segment_access_duration{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2.114125 +# HELP milvus_querynode_segment_access_global_duration global time cost of accessing segments +# TYPE milvus_querynode_segment_access_global_duration histogram +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1"} 3 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="100"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="5000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="20000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="50000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="100000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="250000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="500000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="3.6e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="5e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1e+07"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_querynode_segment_access_global_duration_sum{node_id="2",query_type="query"} 3.8771669999999996 +milvus_querynode_segment_access_global_duration_count{node_id="2",query_type="query"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1"} 1 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="100"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="500"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="5000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="10000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="20000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="50000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="100000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="250000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="500000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="3.6e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="5e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1e+07"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="+Inf"} 2 +milvus_querynode_segment_access_global_duration_sum{node_id="2",query_type="search"} 2.114125 +milvus_querynode_segment_access_global_duration_count{node_id="2",query_type="search"} 2 +# HELP milvus_querynode_segment_access_total number of segments accessed +# TYPE milvus_querynode_segment_access_total counter +milvus_querynode_segment_access_total{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_segment_access_total{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2 +# HELP milvus_querynode_segment_access_wait_cache_duration total time cost of waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_duration counter +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0 +# HELP milvus_querynode_segment_access_wait_cache_global_duration global time cost of waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_global_duration histogram +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="2",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="2",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="2",query_type="search"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="2",query_type="search"} 0 +# HELP milvus_querynode_segment_access_wait_cache_total number of segments waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_total counter +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0 +# HELP milvus_querynode_segment_latency_per_vector one vector's search latency per segment +# TYPE milvus_querynode_segment_latency_per_vector histogram +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="5"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="20"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="50"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="100"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="200"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="300"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="400"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="500"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="600"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="700"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="800"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="900"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1000"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1500"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="3000"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10000"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="60000"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_segment_latency_per_vector_sum{node_id="2",query_type="search",segment_state="Sealed"} 0.5 +milvus_querynode_segment_latency_per_vector_count{node_id="2",query_type="search",segment_state="Sealed"} 2 +# HELP milvus_querynode_segment_num number of segments loaded, clustered by its collection, partition, state and # of indexed fields +# TYPE milvus_querynode_segment_num gauge +milvus_querynode_segment_num{collection_id="454172268311087164",indexed_field_count="1",node_id="2",partition_id="454172268311087165",segment_level="L1",segment_state="Sealed"} 0 +# HELP milvus_querynode_sq_core_latency latency of search or query latency in segcore +# TYPE milvus_querynode_sq_core_latency histogram +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="5"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="10"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="20"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="50"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="100"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="200"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="300"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="400"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="600"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="700"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="800"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="900"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="1500"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="3000"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="60000"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_core_latency_sum{node_id="2",query_type="query"} 2 +milvus_querynode_sq_core_latency_count{node_id="2",query_type="query"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="5"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="10"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="20"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="50"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="100"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="200"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="300"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="400"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="500"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="600"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="700"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="800"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="900"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="1000"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="1500"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="3000"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="10000"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="60000"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_core_latency_sum{node_id="2",query_type="search"} 1 +milvus_querynode_sq_core_latency_count{node_id="2",query_type="search"} 2 +# HELP milvus_querynode_sq_queue_latency latency of search or query in queue +# TYPE milvus_querynode_sq_queue_latency histogram +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="5"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="10"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="20"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="50"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="100"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="200"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="300"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="400"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="500"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="600"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="700"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="800"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="900"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="1000"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="1500"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="3000"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="10000"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="60000"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="+Inf"} 5 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0.040208 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="5"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="10"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="20"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="50"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="100"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="200"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="300"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="400"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="500"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="600"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="700"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="800"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="900"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="1000"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="1500"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="3000"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="10000"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="60000"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="+Inf"} 2 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0.040541 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2 +# HELP milvus_querynode_sq_queue_user_latency latency per user of search or query in queue +# TYPE milvus_querynode_sq_queue_user_latency histogram +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="5"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="10"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="20"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="50"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="100"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="200"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="300"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="400"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="500"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="600"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="700"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="800"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="900"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="1000"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="1500"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="3000"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="10000"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="60000"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="+Inf"} 5 +milvus_querynode_sq_queue_user_latency_sum{node_id="2",query_type="query",username=""} 0.040208 +milvus_querynode_sq_queue_user_latency_count{node_id="2",query_type="query",username=""} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="5"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="10"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="20"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="50"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="100"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="200"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="300"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="400"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="500"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="600"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="700"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="800"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="900"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="1000"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="1500"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="3000"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="10000"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="60000"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="+Inf"} 2 +milvus_querynode_sq_queue_user_latency_sum{node_id="2",query_type="search",username=""} 0.040541 +milvus_querynode_sq_queue_user_latency_count{node_id="2",query_type="search",username=""} 2 +# HELP milvus_querynode_sq_reduce_latency latency of reduce search or query result +# TYPE milvus_querynode_sq_reduce_latency histogram +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="5"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="10"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="20"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="50"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="100"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="200"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="300"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="400"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="500"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="600"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="700"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="800"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="900"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1500"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="3000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="10000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="60000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="5"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="10"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="20"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="50"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="100"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="200"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="300"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="400"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="500"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="600"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="700"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="800"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="900"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1500"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="3000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="10000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="60000"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="5"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="10"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="20"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="50"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="100"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="200"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="300"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="400"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="500"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="600"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="700"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="800"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="900"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1500"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="3000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="10000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="60000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="5"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="10"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="20"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="50"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="100"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="200"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="300"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="400"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="500"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="600"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="700"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="800"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="900"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1500"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="3000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="10000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="60000"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 2 +# HELP milvus_querynode_sq_req_count count of search / query request +# TYPE milvus_querynode_sq_req_count counter +milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="FromLeader",status="success"} 5 +milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="FromLeader",status="total"} 5 +milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="OnLeader",status="success"} 5 +milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="OnLeader",status="total"} 5 +milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="FromLeader",status="success"} 2 +milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="FromLeader",status="total"} 2 +milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="OnLeader",status="success"} 2 +milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="OnLeader",status="total"} 2 +# HELP milvus_querynode_sq_req_latency latency of Search or query requests +# TYPE milvus_querynode_sq_req_latency histogram +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="5"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="10"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="20"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="50"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="100"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="200"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="300"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="400"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="500"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="600"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="700"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="800"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="900"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="1000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="1500"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="3000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="10000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="60000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="2",query_type="query",scope="FromLeader"} 3 +milvus_querynode_sq_req_latency_count{node_id="2",query_type="query",scope="FromLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="5"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="10"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="20"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="50"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="100"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="200"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="300"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="400"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="500"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="600"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="700"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="800"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="900"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="1000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="1500"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="3000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="10000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="60000"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="2",query_type="query",scope="OnLeader"} 1127 +milvus_querynode_sq_req_latency_count{node_id="2",query_type="query",scope="OnLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="5"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="10"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="20"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="50"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="100"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="200"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="300"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="400"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="500"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="600"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="700"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="800"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="900"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="1000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="1500"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="3000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="10000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="60000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="2",query_type="search",scope="FromLeader"} 2 +milvus_querynode_sq_req_latency_count{node_id="2",query_type="search",scope="FromLeader"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="5"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="10"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="20"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="50"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="100"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="200"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="300"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="400"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="500"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="600"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="700"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="800"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="900"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="1000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="1500"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="3000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="10000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="60000"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="2",query_type="search",scope="OnLeader"} 470 +milvus_querynode_sq_req_latency_count{node_id="2",query_type="search",scope="OnLeader"} 2 +# HELP milvus_querynode_sq_segment_latency latency of search or query per segment +# TYPE milvus_querynode_sq_segment_latency histogram +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="5"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="10"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="20"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="50"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="100"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="200"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="300"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="400"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="500"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="600"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="700"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="800"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="900"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="1000"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="1500"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="3000"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="10000"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="60000"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="+Inf"} 5 +milvus_querynode_sq_segment_latency_sum{node_id="2",query_type="query",segment_state="Sealed"} 2 +milvus_querynode_sq_segment_latency_count{node_id="2",query_type="query",segment_state="Sealed"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="5"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="20"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="50"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="100"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="200"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="300"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="400"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="500"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="600"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="700"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="800"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="900"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1000"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1500"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="3000"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10000"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="60000"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_sq_segment_latency_sum{node_id="2",query_type="search",segment_state="Sealed"} 1 +milvus_querynode_sq_segment_latency_count{node_id="2",query_type="search",segment_state="Sealed"} 2 +# HELP milvus_querynode_sq_wait_tsafe_latency latency of search or query to wait for tsafe +# TYPE milvus_querynode_sq_wait_tsafe_latency histogram +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="5"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="10"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="20"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="50"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="100"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="200"} 4 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="300"} 4 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="400"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="500"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="600"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="700"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="800"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="900"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="1000"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="1500"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="3000"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="10000"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="60000"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="2",query_type="query"} 1122 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="2",query_type="query"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="5"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="10"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="20"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="50"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="100"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="200"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="300"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="400"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="500"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="600"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="700"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="800"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="900"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="1000"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="1500"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="3000"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="10000"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="60000"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="2",query_type="search"} 466 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="2",query_type="search"} 2 +# HELP milvus_querynode_wait_processing_msg_count count of wait processing msg +# TYPE milvus_querynode_wait_processing_msg_count gauge +milvus_querynode_wait_processing_msg_count{msg_type="delete",node_id="2"} 0 +milvus_querynode_wait_processing_msg_count{msg_type="insert",node_id="2"} 0 +# HELP milvus_querynode_watch_dml_channel_latency latency of watch dml channel +# TYPE milvus_querynode_watch_dml_channel_latency histogram +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="5"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="10"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="20"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="50"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="100"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="200"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="300"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="400"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="500"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="600"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="700"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="800"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="900"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="1000"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="1500"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="3000"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="10000"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="60000"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="+Inf"} 1 +milvus_querynode_watch_dml_channel_latency_sum{node_id="2"} 0 +milvus_querynode_watch_dml_channel_latency_count{node_id="2"} 1 +# HELP milvus_rootcoord_collection_num number of collections +# TYPE milvus_rootcoord_collection_num gauge +milvus_rootcoord_collection_num{db_name="default"} 0 +# HELP milvus_rootcoord_credential_num number of credentials +# TYPE milvus_rootcoord_credential_num gauge +milvus_rootcoord_credential_num 0 +# HELP milvus_rootcoord_ddl_req_count count of DDL operations +# TYPE milvus_rootcoord_ddl_req_count counter +milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="fail"} 3 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="success"} 15 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="total"} 18 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="success"} 2 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="total"} 2 +milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ListDatabases",status="success"} 2 +milvus_rootcoord_ddl_req_count{function_name="ListDatabases",status="total"} 2 +milvus_rootcoord_ddl_req_count{function_name="PolicyList",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="PolicyList",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowCollections",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowCollections",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="success"} 4 +milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="total"} 4 +# HELP milvus_rootcoord_ddl_req_latency latency of each DDL operations +# TYPE milvus_rootcoord_ddl_req_latency histogram +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="5"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="20"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="50"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="100"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="200"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="300"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="400"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="600"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="700"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="800"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="900"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="3000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="10000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="CreateCollection"} 7 +milvus_rootcoord_ddl_req_latency_count{function_name="CreateCollection"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="5"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="10"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="20"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="50"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="100"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="200"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="300"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="400"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="500"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="600"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="700"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="800"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="900"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1000"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1500"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="3000"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="10000"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="60000"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="+Inf"} 15 +milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeCollection"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeCollection"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="5"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="10"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="20"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="50"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="100"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="200"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="300"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="400"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="500"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="600"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="700"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="800"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="900"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1500"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="3000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="10000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="60000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="+Inf"} 2 +milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeDatabase"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeDatabase"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="5"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="20"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="50"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="100"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="200"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="300"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="400"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="600"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="700"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="800"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="900"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="3000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="10000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="DropCollection"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="DropCollection"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="5"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="10"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="20"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="50"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="100"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="200"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="300"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="400"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="500"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="600"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="700"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="800"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="900"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1500"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="3000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="10000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="60000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="+Inf"} 2 +milvus_rootcoord_ddl_req_latency_sum{function_name="ListDatabases"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ListDatabases"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="5"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="10"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="20"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="50"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="100"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="200"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="300"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="400"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="600"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="700"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="800"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="900"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="3000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="10000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="PolicyList"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="PolicyList"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="5"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="10"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="20"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="50"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="100"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="200"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="300"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="400"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="600"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="700"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="800"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="900"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1500"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="3000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="10000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="ShowCollections"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ShowCollections"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="5"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="10"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="20"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="50"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="100"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="200"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="300"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="400"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="500"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="600"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="700"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="800"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="900"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1000"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1500"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="3000"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="10000"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="60000"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="+Inf"} 4 +milvus_rootcoord_ddl_req_latency_sum{function_name="ShowPartitions"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ShowPartitions"} 4 +# HELP milvus_rootcoord_ddl_req_latency_in_queue latency of each DDL operations in queue +# TYPE milvus_rootcoord_ddl_req_latency_in_queue histogram +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="CreateCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="CreateCollection"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.005"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.01"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.025"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.05"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.1"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.25"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.5"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="1"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="2.5"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="5"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="10"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="+Inf"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="DescribeCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DescribeCollection"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="DropCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DropCollection"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="ShowCollections"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="ShowCollections"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.005"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.01"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.025"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.05"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.1"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.25"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="1"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="2.5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="10"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="+Inf"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="ShowPartitions"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="ShowPartitions"} 4 +# HELP milvus_rootcoord_dml_channel_num number of DML channels +# TYPE milvus_rootcoord_dml_channel_num gauge +milvus_rootcoord_dml_channel_num 16 +# HELP milvus_rootcoord_entity_num number of entities, clustered by collection and their status(loaded/total) +# TYPE milvus_rootcoord_entity_num gauge +milvus_rootcoord_entity_num{collection_name="hello_milvus",status="loaded"} 3001 +milvus_rootcoord_entity_num{collection_name="hello_milvus",status="total"} 3001 +# HELP milvus_rootcoord_force_deny_writing_counter The number of times milvus turns into force-deny-writing states +# TYPE milvus_rootcoord_force_deny_writing_counter counter +milvus_rootcoord_force_deny_writing_counter 0 +# HELP milvus_rootcoord_id_alloc_count count of ID allocated +# TYPE milvus_rootcoord_id_alloc_count counter +milvus_rootcoord_id_alloc_count 400018 +# HELP milvus_rootcoord_indexed_entity_num indexed number of entities, clustered by collection, index name and whether it's a vector index +# TYPE milvus_rootcoord_indexed_entity_num gauge +milvus_rootcoord_indexed_entity_num{collection_name="hello_milvus",index_name="embeddings",is_vector_index="true"} 3001 +# HELP milvus_rootcoord_msgstream_obj_num number of message streams +# TYPE milvus_rootcoord_msgstream_obj_num gauge +milvus_rootcoord_msgstream_obj_num 16 +# HELP milvus_rootcoord_num_of_roles The number of roles +# TYPE milvus_rootcoord_num_of_roles gauge +milvus_rootcoord_num_of_roles 0 +# HELP milvus_rootcoord_partition_num number of partitions +# TYPE milvus_rootcoord_partition_num gauge +milvus_rootcoord_partition_num 0 +# HELP milvus_rootcoord_produce_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_rootcoord_produce_tt_lag_ms gauge +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0"} 150 +# HELP milvus_rootcoord_proxy_num number of proxy nodes managered by rootcoord +# TYPE milvus_rootcoord_proxy_num gauge +milvus_rootcoord_proxy_num 1 +# HELP milvus_rootcoord_qn_mem_high_water_level querynode memory high water level +# TYPE milvus_rootcoord_qn_mem_high_water_level gauge +milvus_rootcoord_qn_mem_high_water_level 0 +# HELP milvus_rootcoord_sync_timetick_latency latency of synchronizing timetick message +# TYPE milvus_rootcoord_sync_timetick_latency histogram +milvus_rootcoord_sync_timetick_latency_bucket{le="5"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="10"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="20"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="50"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="100"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="200"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="300"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="400"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="500"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="600"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="700"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="800"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="900"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="1000"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="1500"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="3000"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="10000"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="60000"} 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="+Inf"} 51 +milvus_rootcoord_sync_timetick_latency_sum 0 +milvus_rootcoord_sync_timetick_latency_count 51 +# HELP milvus_rootcoord_timestamp lateste timestamp allocated in memory +# TYPE milvus_rootcoord_timestamp gauge +milvus_rootcoord_timestamp 4.541724788559708e+17 +# HELP milvus_rootcoord_timestamp_saved timestamp saved in meta storage +# TYPE milvus_rootcoord_timestamp_saved gauge +milvus_rootcoord_timestamp_saved 1.732530515e+09 +# HELP milvus_runtime_info Runtime information of milvus +# TYPE milvus_runtime_info gauge +milvus_runtime_info{meta="etcd",mq="rocksmq"} 1 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 14.19 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 70 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 3.57658624e+08 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.73252970801e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 8.257282048e+09 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes 1.8446744073709552e+19 +# HELP ann_iterator_init_latency ann iterator init latency (ms) +# TYPE ann_iterator_init_latency histogram +ann_iterator_init_latency_bucket{module="cardinal",le="1"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="2"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="4"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="8"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="16"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="32"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="64"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="128"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="256"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="512"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="1024"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="2048"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="4096"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="8192"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="16384"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="32768"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="65536"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="+Inf"} 0 +ann_iterator_init_latency_sum{module="cardinal"} 0 +ann_iterator_init_latency_count{module="cardinal"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="2"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="4"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="8"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="16"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="32"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="64"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="128"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="256"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="512"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1024"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="2048"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="4096"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="8192"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="16384"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="32768"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="65536"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1.048576e+06"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="+Inf"} 0 +ann_iterator_init_latency_sum{module="knowhere"} 0 +ann_iterator_init_latency_count{module="knowhere"} 0 +# HELP bf_search_cnt number of bf search per request +# TYPE bf_search_cnt histogram +bf_search_cnt_bucket{module="cardinal",le="1"} 0 +bf_search_cnt_bucket{module="cardinal",le="2"} 0 +bf_search_cnt_bucket{module="cardinal",le="4"} 0 +bf_search_cnt_bucket{module="cardinal",le="8"} 0 +bf_search_cnt_bucket{module="cardinal",le="16"} 0 +bf_search_cnt_bucket{module="cardinal",le="32"} 0 +bf_search_cnt_bucket{module="cardinal",le="64"} 0 +bf_search_cnt_bucket{module="cardinal",le="128"} 0 +bf_search_cnt_bucket{module="cardinal",le="256"} 0 +bf_search_cnt_bucket{module="cardinal",le="512"} 0 +bf_search_cnt_bucket{module="cardinal",le="1024"} 0 +bf_search_cnt_bucket{module="cardinal",le="2048"} 0 +bf_search_cnt_bucket{module="cardinal",le="4096"} 0 +bf_search_cnt_bucket{module="cardinal",le="8192"} 0 +bf_search_cnt_bucket{module="cardinal",le="16384"} 0 +bf_search_cnt_bucket{module="cardinal",le="32768"} 0 +bf_search_cnt_bucket{module="cardinal",le="65536"} 0 +bf_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +bf_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +bf_search_cnt_sum{module="cardinal"} 0 +bf_search_cnt_count{module="cardinal"} 0 +# HELP bitset_ratio bitset ratio +# TYPE bitset_ratio histogram +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.05"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.1"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.15"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.2"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.25"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.3"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.35"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.4"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.45"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.5"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.55"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.6"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.65"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.7"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.75"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.8"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.85"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.9"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.95"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="1"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="+Inf"} 0 +bitset_ratio_sum{module="PROMETHEUS_LABEL_CARDINAL"} 0 +bitset_ratio_count{module="PROMETHEUS_LABEL_CARDINAL"} 0 +# HELP build_latency index build latency (s) +# TYPE build_latency histogram +build_latency_bucket{module="cardinal",le="1"} 0 +build_latency_bucket{module="cardinal",le="2"} 0 +build_latency_bucket{module="cardinal",le="4"} 0 +build_latency_bucket{module="cardinal",le="8"} 0 +build_latency_bucket{module="cardinal",le="16"} 0 +build_latency_bucket{module="cardinal",le="32"} 0 +build_latency_bucket{module="cardinal",le="64"} 0 +build_latency_bucket{module="cardinal",le="128"} 0 +build_latency_bucket{module="cardinal",le="256"} 0 +build_latency_bucket{module="cardinal",le="512"} 0 +build_latency_bucket{module="cardinal",le="1024"} 0 +build_latency_bucket{module="cardinal",le="2048"} 0 +build_latency_bucket{module="cardinal",le="4096"} 0 +build_latency_bucket{module="cardinal",le="8192"} 0 +build_latency_bucket{module="cardinal",le="16384"} 0 +build_latency_bucket{module="cardinal",le="32768"} 0 +build_latency_bucket{module="cardinal",le="65536"} 0 +build_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +build_latency_bucket{module="cardinal",le="+Inf"} 0 +build_latency_sum{module="cardinal"} 0 +build_latency_count{module="cardinal"} 0 +build_latency_bucket{module="knowhere",le="1"} 1 +build_latency_bucket{module="knowhere",le="2"} 1 +build_latency_bucket{module="knowhere",le="4"} 1 +build_latency_bucket{module="knowhere",le="8"} 1 +build_latency_bucket{module="knowhere",le="16"} 1 +build_latency_bucket{module="knowhere",le="32"} 1 +build_latency_bucket{module="knowhere",le="64"} 1 +build_latency_bucket{module="knowhere",le="128"} 1 +build_latency_bucket{module="knowhere",le="256"} 1 +build_latency_bucket{module="knowhere",le="512"} 1 +build_latency_bucket{module="knowhere",le="1024"} 1 +build_latency_bucket{module="knowhere",le="2048"} 1 +build_latency_bucket{module="knowhere",le="4096"} 1 +build_latency_bucket{module="knowhere",le="8192"} 1 +build_latency_bucket{module="knowhere",le="16384"} 1 +build_latency_bucket{module="knowhere",le="32768"} 1 +build_latency_bucket{module="knowhere",le="65536"} 1 +build_latency_bucket{module="knowhere",le="1.048576e+06"} 1 +build_latency_bucket{module="knowhere",le="+Inf"} 1 +build_latency_sum{module="knowhere"} 0.006943917 +build_latency_count{module="knowhere"} 1 +# HELP cache_hit_cnt cache hit cnt per request +# TYPE cache_hit_cnt histogram +cache_hit_cnt_bucket{module="cardinal",le="1"} 0 +cache_hit_cnt_bucket{module="cardinal",le="2"} 0 +cache_hit_cnt_bucket{module="cardinal",le="4"} 0 +cache_hit_cnt_bucket{module="cardinal",le="8"} 0 +cache_hit_cnt_bucket{module="cardinal",le="16"} 0 +cache_hit_cnt_bucket{module="cardinal",le="32"} 0 +cache_hit_cnt_bucket{module="cardinal",le="64"} 0 +cache_hit_cnt_bucket{module="cardinal",le="128"} 0 +cache_hit_cnt_bucket{module="cardinal",le="256"} 0 +cache_hit_cnt_bucket{module="cardinal",le="512"} 0 +cache_hit_cnt_bucket{module="cardinal",le="1024"} 0 +cache_hit_cnt_bucket{module="cardinal",le="2048"} 0 +cache_hit_cnt_bucket{module="cardinal",le="4096"} 0 +cache_hit_cnt_bucket{module="cardinal",le="8192"} 0 +cache_hit_cnt_bucket{module="cardinal",le="16384"} 0 +cache_hit_cnt_bucket{module="cardinal",le="32768"} 0 +cache_hit_cnt_bucket{module="cardinal",le="65536"} 0 +cache_hit_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +cache_hit_cnt_bucket{module="cardinal",le="+Inf"} 0 +cache_hit_cnt_sum{module="cardinal"} 0 +cache_hit_cnt_count{module="cardinal"} 0 +# HELP diskann_bitset_ratio DISKANN bitset ratio for search and range search +# TYPE diskann_bitset_ratio histogram +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.05"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.1"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.15"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.2"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.25"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.3"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.35"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.4"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.45"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.5"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.55"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.6"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.65"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.7"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.75"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.8"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.85"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.9"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.95"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="1"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +diskann_bitset_ratio_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +diskann_bitset_ratio_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP diskann_range_search_iters DISKANN range search iterations +# TYPE diskann_range_search_iters histogram +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="2"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="4"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="6"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="8"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="10"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="12"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="14"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="16"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="18"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="20"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="22"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +diskann_range_search_iters_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +diskann_range_search_iters_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP diskann_search_hops DISKANN search hops +# TYPE diskann_search_hops histogram +diskann_search_hops_bucket{module="knowhere",le="1"} 0 +diskann_search_hops_bucket{module="knowhere",le="2"} 0 +diskann_search_hops_bucket{module="knowhere",le="4"} 0 +diskann_search_hops_bucket{module="knowhere",le="8"} 0 +diskann_search_hops_bucket{module="knowhere",le="16"} 0 +diskann_search_hops_bucket{module="knowhere",le="32"} 0 +diskann_search_hops_bucket{module="knowhere",le="64"} 0 +diskann_search_hops_bucket{module="knowhere",le="128"} 0 +diskann_search_hops_bucket{module="knowhere",le="256"} 0 +diskann_search_hops_bucket{module="knowhere",le="512"} 0 +diskann_search_hops_bucket{module="knowhere",le="1024"} 0 +diskann_search_hops_bucket{module="knowhere",le="2048"} 0 +diskann_search_hops_bucket{module="knowhere",le="4096"} 0 +diskann_search_hops_bucket{module="knowhere",le="8192"} 0 +diskann_search_hops_bucket{module="knowhere",le="16384"} 0 +diskann_search_hops_bucket{module="knowhere",le="32768"} 0 +diskann_search_hops_bucket{module="knowhere",le="65536"} 0 +diskann_search_hops_bucket{module="knowhere",le="1.048576e+06"} 0 +diskann_search_hops_bucket{module="knowhere",le="+Inf"} 0 +diskann_search_hops_sum{module="knowhere"} 0 +diskann_search_hops_count{module="knowhere"} 0 +# HELP exec_latency execute latency per request +# TYPE exec_latency histogram +exec_latency_bucket{module="cardinal",le="1"} 0 +exec_latency_bucket{module="cardinal",le="2"} 0 +exec_latency_bucket{module="cardinal",le="4"} 0 +exec_latency_bucket{module="cardinal",le="8"} 0 +exec_latency_bucket{module="cardinal",le="16"} 0 +exec_latency_bucket{module="cardinal",le="32"} 0 +exec_latency_bucket{module="cardinal",le="64"} 0 +exec_latency_bucket{module="cardinal",le="128"} 0 +exec_latency_bucket{module="cardinal",le="256"} 0 +exec_latency_bucket{module="cardinal",le="512"} 0 +exec_latency_bucket{module="cardinal",le="1024"} 0 +exec_latency_bucket{module="cardinal",le="2048"} 0 +exec_latency_bucket{module="cardinal",le="4096"} 0 +exec_latency_bucket{module="cardinal",le="8192"} 0 +exec_latency_bucket{module="cardinal",le="16384"} 0 +exec_latency_bucket{module="cardinal",le="32768"} 0 +exec_latency_bucket{module="cardinal",le="65536"} 0 +exec_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +exec_latency_bucket{module="cardinal",le="+Inf"} 0 +exec_latency_sum{module="cardinal"} 0 +exec_latency_count{module="cardinal"} 0 +# HELP filter_connectivity_ratio avg connectivity ratio set under filtering per request +# TYPE filter_connectivity_ratio histogram +filter_connectivity_ratio_bucket{module="cardinal",le="1"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="2"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="4"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="8"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="16"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="32"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="64"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="128"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="256"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="512"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="1024"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="2048"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="4096"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="8192"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="16384"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="32768"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="65536"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="+Inf"} 0 +filter_connectivity_ratio_sum{module="cardinal"} 0 +filter_connectivity_ratio_count{module="cardinal"} 0 +# HELP filter_mv_activated_fields_cnt avg mv activated fields per request +# TYPE filter_mv_activated_fields_cnt histogram +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_activated_fields_cnt_sum{module="cardinal"} 0 +filter_mv_activated_fields_cnt_count{module="cardinal"} 0 +# HELP filter_mv_change_base_cnt mv change base cnt per request +# TYPE filter_mv_change_base_cnt histogram +filter_mv_change_base_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_change_base_cnt_sum{module="cardinal"} 0 +filter_mv_change_base_cnt_count{module="cardinal"} 0 +# HELP filter_mv_only_cnt mv only cnt per request +# TYPE filter_mv_only_cnt histogram +filter_mv_only_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_only_cnt_sum{module="cardinal"} 0 +filter_mv_only_cnt_count{module="cardinal"} 0 +# HELP filter_mv_supplement_ep_bool_cnt mv supplement ep from bitset boolean cnt per request +# TYPE filter_mv_supplement_ep_bool_cnt histogram +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_supplement_ep_bool_cnt_sum{module="cardinal"} 0 +filter_mv_supplement_ep_bool_cnt_count{module="cardinal"} 0 +# HELP graph_search_cnt number of graph search per request +# TYPE graph_search_cnt histogram +graph_search_cnt_bucket{module="cardinal",le="1"} 0 +graph_search_cnt_bucket{module="cardinal",le="2"} 0 +graph_search_cnt_bucket{module="cardinal",le="4"} 0 +graph_search_cnt_bucket{module="cardinal",le="8"} 0 +graph_search_cnt_bucket{module="cardinal",le="16"} 0 +graph_search_cnt_bucket{module="cardinal",le="32"} 0 +graph_search_cnt_bucket{module="cardinal",le="64"} 0 +graph_search_cnt_bucket{module="cardinal",le="128"} 0 +graph_search_cnt_bucket{module="cardinal",le="256"} 0 +graph_search_cnt_bucket{module="cardinal",le="512"} 0 +graph_search_cnt_bucket{module="cardinal",le="1024"} 0 +graph_search_cnt_bucket{module="cardinal",le="2048"} 0 +graph_search_cnt_bucket{module="cardinal",le="4096"} 0 +graph_search_cnt_bucket{module="cardinal",le="8192"} 0 +graph_search_cnt_bucket{module="cardinal",le="16384"} 0 +graph_search_cnt_bucket{module="cardinal",le="32768"} 0 +graph_search_cnt_bucket{module="cardinal",le="65536"} 0 +graph_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +graph_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +graph_search_cnt_sum{module="cardinal"} 0 +graph_search_cnt_count{module="cardinal"} 0 +# HELP hnsw_bitset_ratio HNSW bitset ratio for search and range search +# TYPE hnsw_bitset_ratio histogram +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.05"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.1"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.15"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.2"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.25"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.3"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.35"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.4"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.45"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.5"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.55"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.6"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.65"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.7"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.75"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.8"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.85"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.9"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.95"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="1"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +hnsw_bitset_ratio_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +hnsw_bitset_ratio_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP hnsw_search_hops HNSW search hops in layer 0 +# TYPE hnsw_search_hops histogram +hnsw_search_hops_bucket{module="knowhere",le="1"} 0 +hnsw_search_hops_bucket{module="knowhere",le="2"} 0 +hnsw_search_hops_bucket{module="knowhere",le="4"} 0 +hnsw_search_hops_bucket{module="knowhere",le="8"} 0 +hnsw_search_hops_bucket{module="knowhere",le="16"} 0 +hnsw_search_hops_bucket{module="knowhere",le="32"} 0 +hnsw_search_hops_bucket{module="knowhere",le="64"} 0 +hnsw_search_hops_bucket{module="knowhere",le="128"} 0 +hnsw_search_hops_bucket{module="knowhere",le="256"} 0 +hnsw_search_hops_bucket{module="knowhere",le="512"} 0 +hnsw_search_hops_bucket{module="knowhere",le="1024"} 0 +hnsw_search_hops_bucket{module="knowhere",le="2048"} 0 +hnsw_search_hops_bucket{module="knowhere",le="4096"} 0 +hnsw_search_hops_bucket{module="knowhere",le="8192"} 0 +hnsw_search_hops_bucket{module="knowhere",le="16384"} 0 +hnsw_search_hops_bucket{module="knowhere",le="32768"} 0 +hnsw_search_hops_bucket{module="knowhere",le="65536"} 0 +hnsw_search_hops_bucket{module="knowhere",le="1.048576e+06"} 0 +hnsw_search_hops_bucket{module="knowhere",le="+Inf"} 0 +hnsw_search_hops_sum{module="knowhere"} 0 +hnsw_search_hops_count{module="knowhere"} 0 +# HELP internal_core_search_latency [cpp]latency(us) of search on segment +# TYPE internal_core_search_latency histogram +internal_core_search_latency_bucket{type="scalar_latency",le="1"} 0 +internal_core_search_latency_bucket{type="scalar_latency",le="2"} 0 +internal_core_search_latency_bucket{type="scalar_latency",le="4"} 0 +internal_core_search_latency_bucket{type="scalar_latency",le="8"} 1 +internal_core_search_latency_bucket{type="scalar_latency",le="16"} 1 +internal_core_search_latency_bucket{type="scalar_latency",le="32"} 1 +internal_core_search_latency_bucket{type="scalar_latency",le="64"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="128"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="256"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="512"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="1024"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="2048"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="4096"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="8192"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="16384"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="32768"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="65536"} 2 +internal_core_search_latency_bucket{type="scalar_latency",le="+Inf"} 2 +internal_core_search_latency_sum{type="scalar_latency"} 57.667 +internal_core_search_latency_count{type="scalar_latency"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.05"} 1 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.1"} 1 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.15"} 1 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.2"} 1 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.25"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.3"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.35"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.4"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.45"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.5"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.55"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.6"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.65"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.7"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.75"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.8"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.85"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.9"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.95"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="1"} 2 +internal_core_search_latency_bucket{type="scalar_proportion",le="+Inf"} 2 +internal_core_search_latency_sum{type="scalar_proportion"} 0.2503988069921382 +internal_core_search_latency_count{type="scalar_proportion"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="1"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="2"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="4"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="8"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="16"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="32"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="64"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="128"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="256"} 1 +internal_core_search_latency_bucket{type="vector_latency",le="512"} 1 +internal_core_search_latency_bucket{type="vector_latency",le="1024"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="2048"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="4096"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="8192"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="16384"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="32768"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="65536"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="+Inf"} 2 +internal_core_search_latency_sum{type="vector_latency"} 986.25 +internal_core_search_latency_count{type="vector_latency"} 2 +# HELP internal_mmap_allocated_space_bytes [cpp]mmap allocated space stats +# TYPE internal_mmap_allocated_space_bytes histogram +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1024"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="8192"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="65536"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="262144"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="524288"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.048576e+06"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="4.194304e+06"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="8.388608e+06"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.6777216e+07"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="6.7108864e+07"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.34217728e+08"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="2.68435456e+08"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="5.36870912e+08"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.073741824e+09"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="+Inf"} 2 +internal_mmap_allocated_space_bytes_sum{type="anon"} 46908 +internal_mmap_allocated_space_bytes_count{type="anon"} 2 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1024"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="8192"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="65536"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="262144"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="524288"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.048576e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="4.194304e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="8.388608e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.6777216e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="6.7108864e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.34217728e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="2.68435456e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="5.36870912e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.073741824e+09"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="+Inf"} 0 +internal_mmap_allocated_space_bytes_sum{type="file"} 0 +internal_mmap_allocated_space_bytes_count{type="file"} 0 +# HELP internal_mmap_in_used_space_bytes [cpp]mmap in used space stats +# TYPE internal_mmap_in_used_space_bytes gauge +internal_mmap_in_used_space_bytes{type="anon"} 0 +internal_mmap_in_used_space_bytes{type="file"} 0 +# HELP internal_storage_kv_size [cpp]kv size stats +# TYPE internal_storage_kv_size histogram +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="1"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="2"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="4"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="8"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="16"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="32"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="64"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="128"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="256"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="512"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="1024"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="2048"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="4096"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="8192"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="16384"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="32768"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="65536"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="+Inf"} 0 +internal_storage_kv_size_sum{persistent_data_op_type="get"} 0 +internal_storage_kv_size_count{persistent_data_op_type="get"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="1"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="2"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="4"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="8"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="16"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="32"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="64"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="128"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="256"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="512"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="1024"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="2048"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="4096"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="8192"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="16384"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="32768"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="65536"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="+Inf"} 0 +internal_storage_kv_size_sum{persistent_data_op_type="put"} 0 +internal_storage_kv_size_count{persistent_data_op_type="put"} 0 +# HELP internal_storage_load_duration [cpp]durations of load segment +# TYPE internal_storage_load_duration histogram +internal_storage_load_duration_bucket{type="deserialize",le="1"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="2"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="4"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="8"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="16"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="32"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="64"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="128"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="256"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="512"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="1024"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="2048"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="4096"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="8192"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="16384"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="32768"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="65536"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="+Inf"} 0 +internal_storage_load_duration_sum{type="deserialize"} 0 +internal_storage_load_duration_count{type="deserialize"} 0 +internal_storage_load_duration_bucket{type="download",le="1"} 0 +internal_storage_load_duration_bucket{type="download",le="2"} 0 +internal_storage_load_duration_bucket{type="download",le="4"} 0 +internal_storage_load_duration_bucket{type="download",le="8"} 0 +internal_storage_load_duration_bucket{type="download",le="16"} 0 +internal_storage_load_duration_bucket{type="download",le="32"} 0 +internal_storage_load_duration_bucket{type="download",le="64"} 0 +internal_storage_load_duration_bucket{type="download",le="128"} 0 +internal_storage_load_duration_bucket{type="download",le="256"} 0 +internal_storage_load_duration_bucket{type="download",le="512"} 0 +internal_storage_load_duration_bucket{type="download",le="1024"} 0 +internal_storage_load_duration_bucket{type="download",le="2048"} 0 +internal_storage_load_duration_bucket{type="download",le="4096"} 0 +internal_storage_load_duration_bucket{type="download",le="8192"} 0 +internal_storage_load_duration_bucket{type="download",le="16384"} 0 +internal_storage_load_duration_bucket{type="download",le="32768"} 0 +internal_storage_load_duration_bucket{type="download",le="65536"} 0 +internal_storage_load_duration_bucket{type="download",le="+Inf"} 0 +internal_storage_load_duration_sum{type="download"} 0 +internal_storage_load_duration_count{type="download"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="1"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="2"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="4"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="8"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="16"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="32"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="64"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="128"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="256"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="512"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="1024"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="2048"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="4096"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="8192"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="16384"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="32768"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="65536"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="+Inf"} 0 +internal_storage_load_duration_sum{type="write_disk"} 0 +internal_storage_load_duration_count{type="write_disk"} 0 +# HELP internal_storage_op_count [cpp]count of persistent data operation +# TYPE internal_storage_op_count counter +internal_storage_op_count{persistent_data_op_type="get"} 0 +internal_storage_op_count{persistent_data_op_type="get",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="list",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="list",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="put",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="put",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="remove",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="remove",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="stat",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="stat",status="success"} 0 +# HELP internal_storage_request_latency [cpp]request latency(ms) on the client side +# TYPE internal_storage_request_latency histogram +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="get"} 0 +internal_storage_request_latency_count{persistent_data_op_type="get"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="list"} 0 +internal_storage_request_latency_count{persistent_data_op_type="list"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="put"} 0 +internal_storage_request_latency_count{persistent_data_op_type="put"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="remove"} 0 +internal_storage_request_latency_count{persistent_data_op_type="remove"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="stat"} 0 +internal_storage_request_latency_count{persistent_data_op_type="stat"} 0 +# HELP io_cnt io cnt per request +# TYPE io_cnt histogram +io_cnt_bucket{module="cardinal",le="1"} 0 +io_cnt_bucket{module="cardinal",le="2"} 0 +io_cnt_bucket{module="cardinal",le="4"} 0 +io_cnt_bucket{module="cardinal",le="8"} 0 +io_cnt_bucket{module="cardinal",le="16"} 0 +io_cnt_bucket{module="cardinal",le="32"} 0 +io_cnt_bucket{module="cardinal",le="64"} 0 +io_cnt_bucket{module="cardinal",le="128"} 0 +io_cnt_bucket{module="cardinal",le="256"} 0 +io_cnt_bucket{module="cardinal",le="512"} 0 +io_cnt_bucket{module="cardinal",le="1024"} 0 +io_cnt_bucket{module="cardinal",le="2048"} 0 +io_cnt_bucket{module="cardinal",le="4096"} 0 +io_cnt_bucket{module="cardinal",le="8192"} 0 +io_cnt_bucket{module="cardinal",le="16384"} 0 +io_cnt_bucket{module="cardinal",le="32768"} 0 +io_cnt_bucket{module="cardinal",le="65536"} 0 +io_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +io_cnt_bucket{module="cardinal",le="+Inf"} 0 +io_cnt_sum{module="cardinal"} 0 +io_cnt_count{module="cardinal"} 0 +# HELP ivf_search_cnt number of ivf search per request +# TYPE ivf_search_cnt histogram +ivf_search_cnt_bucket{module="cardinal",le="1"} 0 +ivf_search_cnt_bucket{module="cardinal",le="2"} 0 +ivf_search_cnt_bucket{module="cardinal",le="4"} 0 +ivf_search_cnt_bucket{module="cardinal",le="8"} 0 +ivf_search_cnt_bucket{module="cardinal",le="16"} 0 +ivf_search_cnt_bucket{module="cardinal",le="32"} 0 +ivf_search_cnt_bucket{module="cardinal",le="64"} 0 +ivf_search_cnt_bucket{module="cardinal",le="128"} 0 +ivf_search_cnt_bucket{module="cardinal",le="256"} 0 +ivf_search_cnt_bucket{module="cardinal",le="512"} 0 +ivf_search_cnt_bucket{module="cardinal",le="1024"} 0 +ivf_search_cnt_bucket{module="cardinal",le="2048"} 0 +ivf_search_cnt_bucket{module="cardinal",le="4096"} 0 +ivf_search_cnt_bucket{module="cardinal",le="8192"} 0 +ivf_search_cnt_bucket{module="cardinal",le="16384"} 0 +ivf_search_cnt_bucket{module="cardinal",le="32768"} 0 +ivf_search_cnt_bucket{module="cardinal",le="65536"} 0 +ivf_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +ivf_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +ivf_search_cnt_sum{module="cardinal"} 0 +ivf_search_cnt_count{module="cardinal"} 0 +# HELP load_latency index load latency (ms) +# TYPE load_latency histogram +load_latency_bucket{module="cardinal",le="1"} 0 +load_latency_bucket{module="cardinal",le="2"} 0 +load_latency_bucket{module="cardinal",le="4"} 0 +load_latency_bucket{module="cardinal",le="8"} 0 +load_latency_bucket{module="cardinal",le="16"} 0 +load_latency_bucket{module="cardinal",le="32"} 0 +load_latency_bucket{module="cardinal",le="64"} 0 +load_latency_bucket{module="cardinal",le="128"} 0 +load_latency_bucket{module="cardinal",le="256"} 0 +load_latency_bucket{module="cardinal",le="512"} 0 +load_latency_bucket{module="cardinal",le="1024"} 0 +load_latency_bucket{module="cardinal",le="2048"} 0 +load_latency_bucket{module="cardinal",le="4096"} 0 +load_latency_bucket{module="cardinal",le="8192"} 0 +load_latency_bucket{module="cardinal",le="16384"} 0 +load_latency_bucket{module="cardinal",le="32768"} 0 +load_latency_bucket{module="cardinal",le="65536"} 0 +load_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +load_latency_bucket{module="cardinal",le="+Inf"} 0 +load_latency_sum{module="cardinal"} 0 +load_latency_count{module="cardinal"} 0 +load_latency_bucket{module="knowhere",le="1"} 1 +load_latency_bucket{module="knowhere",le="2"} 1 +load_latency_bucket{module="knowhere",le="4"} 1 +load_latency_bucket{module="knowhere",le="8"} 1 +load_latency_bucket{module="knowhere",le="16"} 1 +load_latency_bucket{module="knowhere",le="32"} 1 +load_latency_bucket{module="knowhere",le="64"} 1 +load_latency_bucket{module="knowhere",le="128"} 1 +load_latency_bucket{module="knowhere",le="256"} 1 +load_latency_bucket{module="knowhere",le="512"} 1 +load_latency_bucket{module="knowhere",le="1024"} 1 +load_latency_bucket{module="knowhere",le="2048"} 1 +load_latency_bucket{module="knowhere",le="4096"} 1 +load_latency_bucket{module="knowhere",le="8192"} 1 +load_latency_bucket{module="knowhere",le="16384"} 1 +load_latency_bucket{module="knowhere",le="32768"} 1 +load_latency_bucket{module="knowhere",le="65536"} 1 +load_latency_bucket{module="knowhere",le="1.048576e+06"} 1 +load_latency_bucket{module="knowhere",le="+Inf"} 1 +load_latency_sum{module="knowhere"} 0.5315420000000001 +load_latency_count{module="knowhere"} 1 +# HELP quant_compute_cnt quant compute cnt per request +# TYPE quant_compute_cnt histogram +quant_compute_cnt_bucket{module="cardinal",le="1"} 0 +quant_compute_cnt_bucket{module="cardinal",le="2"} 0 +quant_compute_cnt_bucket{module="cardinal",le="4"} 0 +quant_compute_cnt_bucket{module="cardinal",le="8"} 0 +quant_compute_cnt_bucket{module="cardinal",le="16"} 0 +quant_compute_cnt_bucket{module="cardinal",le="32"} 0 +quant_compute_cnt_bucket{module="cardinal",le="64"} 0 +quant_compute_cnt_bucket{module="cardinal",le="128"} 0 +quant_compute_cnt_bucket{module="cardinal",le="256"} 0 +quant_compute_cnt_bucket{module="cardinal",le="512"} 0 +quant_compute_cnt_bucket{module="cardinal",le="1024"} 0 +quant_compute_cnt_bucket{module="cardinal",le="2048"} 0 +quant_compute_cnt_bucket{module="cardinal",le="4096"} 0 +quant_compute_cnt_bucket{module="cardinal",le="8192"} 0 +quant_compute_cnt_bucket{module="cardinal",le="16384"} 0 +quant_compute_cnt_bucket{module="cardinal",le="32768"} 0 +quant_compute_cnt_bucket{module="cardinal",le="65536"} 0 +quant_compute_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +quant_compute_cnt_bucket{module="cardinal",le="+Inf"} 0 +quant_compute_cnt_sum{module="cardinal"} 0 +quant_compute_cnt_count{module="cardinal"} 0 +# HELP queue_latency queue latency per request +# TYPE queue_latency histogram +queue_latency_bucket{module="cardinal",le="1"} 0 +queue_latency_bucket{module="cardinal",le="2"} 0 +queue_latency_bucket{module="cardinal",le="4"} 0 +queue_latency_bucket{module="cardinal",le="8"} 0 +queue_latency_bucket{module="cardinal",le="16"} 0 +queue_latency_bucket{module="cardinal",le="32"} 0 +queue_latency_bucket{module="cardinal",le="64"} 0 +queue_latency_bucket{module="cardinal",le="128"} 0 +queue_latency_bucket{module="cardinal",le="256"} 0 +queue_latency_bucket{module="cardinal",le="512"} 0 +queue_latency_bucket{module="cardinal",le="1024"} 0 +queue_latency_bucket{module="cardinal",le="2048"} 0 +queue_latency_bucket{module="cardinal",le="4096"} 0 +queue_latency_bucket{module="cardinal",le="8192"} 0 +queue_latency_bucket{module="cardinal",le="16384"} 0 +queue_latency_bucket{module="cardinal",le="32768"} 0 +queue_latency_bucket{module="cardinal",le="65536"} 0 +queue_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +queue_latency_bucket{module="cardinal",le="+Inf"} 0 +queue_latency_sum{module="cardinal"} 0 +queue_latency_count{module="cardinal"} 0 +# HELP range_search_latency range search latency (ms) +# TYPE range_search_latency histogram +range_search_latency_bucket{module="cardinal",le="1"} 0 +range_search_latency_bucket{module="cardinal",le="2"} 0 +range_search_latency_bucket{module="cardinal",le="4"} 0 +range_search_latency_bucket{module="cardinal",le="8"} 0 +range_search_latency_bucket{module="cardinal",le="16"} 0 +range_search_latency_bucket{module="cardinal",le="32"} 0 +range_search_latency_bucket{module="cardinal",le="64"} 0 +range_search_latency_bucket{module="cardinal",le="128"} 0 +range_search_latency_bucket{module="cardinal",le="256"} 0 +range_search_latency_bucket{module="cardinal",le="512"} 0 +range_search_latency_bucket{module="cardinal",le="1024"} 0 +range_search_latency_bucket{module="cardinal",le="2048"} 0 +range_search_latency_bucket{module="cardinal",le="4096"} 0 +range_search_latency_bucket{module="cardinal",le="8192"} 0 +range_search_latency_bucket{module="cardinal",le="16384"} 0 +range_search_latency_bucket{module="cardinal",le="32768"} 0 +range_search_latency_bucket{module="cardinal",le="65536"} 0 +range_search_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +range_search_latency_bucket{module="cardinal",le="+Inf"} 0 +range_search_latency_sum{module="cardinal"} 0 +range_search_latency_count{module="cardinal"} 0 +range_search_latency_bucket{module="knowhere",le="1"} 0 +range_search_latency_bucket{module="knowhere",le="2"} 0 +range_search_latency_bucket{module="knowhere",le="4"} 0 +range_search_latency_bucket{module="knowhere",le="8"} 0 +range_search_latency_bucket{module="knowhere",le="16"} 0 +range_search_latency_bucket{module="knowhere",le="32"} 0 +range_search_latency_bucket{module="knowhere",le="64"} 0 +range_search_latency_bucket{module="knowhere",le="128"} 0 +range_search_latency_bucket{module="knowhere",le="256"} 0 +range_search_latency_bucket{module="knowhere",le="512"} 0 +range_search_latency_bucket{module="knowhere",le="1024"} 0 +range_search_latency_bucket{module="knowhere",le="2048"} 0 +range_search_latency_bucket{module="knowhere",le="4096"} 0 +range_search_latency_bucket{module="knowhere",le="8192"} 0 +range_search_latency_bucket{module="knowhere",le="16384"} 0 +range_search_latency_bucket{module="knowhere",le="32768"} 0 +range_search_latency_bucket{module="knowhere",le="65536"} 0 +range_search_latency_bucket{module="knowhere",le="1.048576e+06"} 0 +range_search_latency_bucket{module="knowhere",le="+Inf"} 0 +range_search_latency_sum{module="knowhere"} 0 +range_search_latency_count{module="knowhere"} 0 +# HELP raw_compute_cnt raw compute cnt per request +# TYPE raw_compute_cnt histogram +raw_compute_cnt_bucket{module="cardinal",le="1"} 0 +raw_compute_cnt_bucket{module="cardinal",le="2"} 0 +raw_compute_cnt_bucket{module="cardinal",le="4"} 0 +raw_compute_cnt_bucket{module="cardinal",le="8"} 0 +raw_compute_cnt_bucket{module="cardinal",le="16"} 0 +raw_compute_cnt_bucket{module="cardinal",le="32"} 0 +raw_compute_cnt_bucket{module="cardinal",le="64"} 0 +raw_compute_cnt_bucket{module="cardinal",le="128"} 0 +raw_compute_cnt_bucket{module="cardinal",le="256"} 0 +raw_compute_cnt_bucket{module="cardinal",le="512"} 0 +raw_compute_cnt_bucket{module="cardinal",le="1024"} 0 +raw_compute_cnt_bucket{module="cardinal",le="2048"} 0 +raw_compute_cnt_bucket{module="cardinal",le="4096"} 0 +raw_compute_cnt_bucket{module="cardinal",le="8192"} 0 +raw_compute_cnt_bucket{module="cardinal",le="16384"} 0 +raw_compute_cnt_bucket{module="cardinal",le="32768"} 0 +raw_compute_cnt_bucket{module="cardinal",le="65536"} 0 +raw_compute_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +raw_compute_cnt_bucket{module="cardinal",le="+Inf"} 0 +raw_compute_cnt_sum{module="cardinal"} 0 +raw_compute_cnt_count{module="cardinal"} 0 +# HELP re_search_cnt number of fallback search per request +# TYPE re_search_cnt histogram +re_search_cnt_bucket{module="cardinal",le="1"} 0 +re_search_cnt_bucket{module="cardinal",le="2"} 0 +re_search_cnt_bucket{module="cardinal",le="4"} 0 +re_search_cnt_bucket{module="cardinal",le="8"} 0 +re_search_cnt_bucket{module="cardinal",le="16"} 0 +re_search_cnt_bucket{module="cardinal",le="32"} 0 +re_search_cnt_bucket{module="cardinal",le="64"} 0 +re_search_cnt_bucket{module="cardinal",le="128"} 0 +re_search_cnt_bucket{module="cardinal",le="256"} 0 +re_search_cnt_bucket{module="cardinal",le="512"} 0 +re_search_cnt_bucket{module="cardinal",le="1024"} 0 +re_search_cnt_bucket{module="cardinal",le="2048"} 0 +re_search_cnt_bucket{module="cardinal",le="4096"} 0 +re_search_cnt_bucket{module="cardinal",le="8192"} 0 +re_search_cnt_bucket{module="cardinal",le="16384"} 0 +re_search_cnt_bucket{module="cardinal",le="32768"} 0 +re_search_cnt_bucket{module="cardinal",le="65536"} 0 +re_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +re_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +re_search_cnt_sum{module="cardinal"} 0 +re_search_cnt_count{module="cardinal"} 0 +# HELP search_latency search latency (ms) +# TYPE search_latency histogram +search_latency_bucket{module="cardinal",le="1"} 0 +search_latency_bucket{module="cardinal",le="2"} 0 +search_latency_bucket{module="cardinal",le="4"} 0 +search_latency_bucket{module="cardinal",le="8"} 0 +search_latency_bucket{module="cardinal",le="16"} 0 +search_latency_bucket{module="cardinal",le="32"} 0 +search_latency_bucket{module="cardinal",le="64"} 0 +search_latency_bucket{module="cardinal",le="128"} 0 +search_latency_bucket{module="cardinal",le="256"} 0 +search_latency_bucket{module="cardinal",le="512"} 0 +search_latency_bucket{module="cardinal",le="1024"} 0 +search_latency_bucket{module="cardinal",le="2048"} 0 +search_latency_bucket{module="cardinal",le="4096"} 0 +search_latency_bucket{module="cardinal",le="8192"} 0 +search_latency_bucket{module="cardinal",le="16384"} 0 +search_latency_bucket{module="cardinal",le="32768"} 0 +search_latency_bucket{module="cardinal",le="65536"} 0 +search_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +search_latency_bucket{module="cardinal",le="+Inf"} 0 +search_latency_sum{module="cardinal"} 0 +search_latency_count{module="cardinal"} 0 +search_latency_bucket{module="knowhere",le="1"} 2 +search_latency_bucket{module="knowhere",le="2"} 2 +search_latency_bucket{module="knowhere",le="4"} 2 +search_latency_bucket{module="knowhere",le="8"} 2 +search_latency_bucket{module="knowhere",le="16"} 2 +search_latency_bucket{module="knowhere",le="32"} 2 +search_latency_bucket{module="knowhere",le="64"} 2 +search_latency_bucket{module="knowhere",le="128"} 2 +search_latency_bucket{module="knowhere",le="256"} 2 +search_latency_bucket{module="knowhere",le="512"} 2 +search_latency_bucket{module="knowhere",le="1024"} 2 +search_latency_bucket{module="knowhere",le="2048"} 2 +search_latency_bucket{module="knowhere",le="4096"} 2 +search_latency_bucket{module="knowhere",le="8192"} 2 +search_latency_bucket{module="knowhere",le="16384"} 2 +search_latency_bucket{module="knowhere",le="32768"} 2 +search_latency_bucket{module="knowhere",le="65536"} 2 +search_latency_bucket{module="knowhere",le="1.048576e+06"} 2 +search_latency_bucket{module="knowhere",le="+Inf"} 2 +search_latency_sum{module="knowhere"} 0.759957 +search_latency_count{module="knowhere"} 2 +# HELP search_topk search topk +# TYPE search_topk histogram +search_topk_bucket{module="cardinal",le="1"} 0 +search_topk_bucket{module="cardinal",le="2"} 0 +search_topk_bucket{module="cardinal",le="4"} 0 +search_topk_bucket{module="cardinal",le="8"} 0 +search_topk_bucket{module="cardinal",le="16"} 0 +search_topk_bucket{module="cardinal",le="32"} 0 +search_topk_bucket{module="cardinal",le="64"} 0 +search_topk_bucket{module="cardinal",le="128"} 0 +search_topk_bucket{module="cardinal",le="256"} 0 +search_topk_bucket{module="cardinal",le="512"} 0 +search_topk_bucket{module="cardinal",le="1024"} 0 +search_topk_bucket{module="cardinal",le="2048"} 0 +search_topk_bucket{module="cardinal",le="4096"} 0 +search_topk_bucket{module="cardinal",le="8192"} 0 +search_topk_bucket{module="cardinal",le="16384"} 0 +search_topk_bucket{module="cardinal",le="32768"} 0 +search_topk_bucket{module="cardinal",le="65536"} 0 +search_topk_bucket{module="cardinal",le="1.048576e+06"} 0 +search_topk_bucket{module="cardinal",le="+Inf"} 0 +search_topk_sum{module="cardinal"} 0 +search_topk_count{module="cardinal"} 0 +search_topk_bucket{module="knowhere",le="1"} 0 +search_topk_bucket{module="knowhere",le="2"} 0 +search_topk_bucket{module="knowhere",le="4"} 2 +search_topk_bucket{module="knowhere",le="8"} 2 +search_topk_bucket{module="knowhere",le="16"} 2 +search_topk_bucket{module="knowhere",le="32"} 2 +search_topk_bucket{module="knowhere",le="64"} 2 +search_topk_bucket{module="knowhere",le="128"} 2 +search_topk_bucket{module="knowhere",le="256"} 2 +search_topk_bucket{module="knowhere",le="512"} 2 +search_topk_bucket{module="knowhere",le="1024"} 2 +search_topk_bucket{module="knowhere",le="2048"} 2 +search_topk_bucket{module="knowhere",le="4096"} 2 +search_topk_bucket{module="knowhere",le="8192"} 2 +search_topk_bucket{module="knowhere",le="16384"} 2 +search_topk_bucket{module="knowhere",le="32768"} 2 +search_topk_bucket{module="knowhere",le="65536"} 2 +search_topk_bucket{module="knowhere",le="1.048576e+06"} 2 +search_topk_bucket{module="knowhere",le="+Inf"} 2 +search_topk_sum{module="knowhere"} 6 +search_topk_count{module="knowhere"} 2 From f783bfb1e512abf0777011aaf53f3054a407064b Mon Sep 17 00:00:00 2001 From: David Kirov Date: Wed, 27 Nov 2024 13:17:24 +0100 Subject: [PATCH 12/31] Add tests --- milvus/datadog_checks/milvus/check.py | 20 +- milvus/datadog_checks/milvus/metrics.py | 159 ++++++++ milvus/metadata.csv | 348 ++++++++++-------- milvus/tests/common.py | 178 +++++++++ milvus/tests/conftest.py | 21 +- milvus/tests/docker/Caddyfile | 4 + milvus/tests/docker/docker-compose.yaml | 11 + ...{docker_payload.txt => milvus_payload.txt} | 0 milvus/tests/test_unit.py | 28 +- 9 files changed, 596 insertions(+), 173 deletions(-) create mode 100644 milvus/datadog_checks/milvus/metrics.py create mode 100644 milvus/tests/common.py create mode 100644 milvus/tests/docker/Caddyfile create mode 100644 milvus/tests/docker/docker-compose.yaml rename milvus/tests/fixtures/{docker_payload.txt => milvus_payload.txt} (100%) diff --git a/milvus/datadog_checks/milvus/check.py b/milvus/datadog_checks/milvus/check.py index cc3612d85463a..707f6ae7cf88f 100644 --- a/milvus/datadog_checks/milvus/check.py +++ b/milvus/datadog_checks/milvus/check.py @@ -1,23 +1,21 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Any # noqa: F401 +from datadog_checks.milvus.metrics import METRIC_MAP, RENAME_LABELS_MAP from datadog_checks.base import OpenMetricsBaseCheckV2 -# from datadog_checks.base.utils.db import QueryManager -# from requests.exceptions import ConnectionError, HTTPError, InvalidURL, Timeout -# from json import JSONDecodeError - class MilvusCheck(OpenMetricsBaseCheckV2): - - # This will be the prefix of every metric and service check the integration sends __NAMESPACE__ = 'milvus' - def __init__(self, name, init_config, instances): - super(MilvusCheck, self).__init__(name, init_config, instances) + DEFAULT_METRIC_LIMIT = 0 - # Use self.instance to read the check configuration - # self.url = self.instance.get("url") + def __init__(self, name, init_config, instances=None): + super(MilvusCheck, self).__init__(name, init_config, instances) + def get_default_config(self): + return { + 'metrics': [METRIC_MAP], + 'rename_labels': RENAME_LABELS_MAP, + } diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py new file mode 100644 index 0000000000000..24b40a7fc7218 --- /dev/null +++ b/milvus/datadog_checks/milvus/metrics.py @@ -0,0 +1,159 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +METRIC_MAP = { + 'milvus_build_info': 'build_info', + 'milvus_cgo_active_future_total': 'cgo.active_future_total', + 'milvus_cgo_cgo_duration_seconds': 'cgo.cgo_duration_seconds', + 'milvus_cgo_cgo_queue_duration_seconds': 'cgo.cgo_queue_duration_seconds', + 'milvus_cgo_running_cgo_call_total': 'cgo.running_cgo_call_total', + 'milvus_datacoord_channel_checkpoint_unix_seconds': 'datacoord.channel_checkpoint_unix_seconds', + 'milvus_datacoord_collection_num': 'datacoord.collection_num', + 'milvus_datacoord_consume_datanode_tt_lag_ms': 'datacoord.consume_datanode_tt_lag_ms', + 'milvus_datacoord_datanode_num': 'datacoord.datanode_num', + 'milvus_datacoord_import_tasks': 'datacoord.import_tasks', + 'milvus_datacoord_index_node_num': 'datacoord.index.node_num', + 'milvus_datacoord_index_req_count': 'datacoord.index.req_count', + 'milvus_datacoord_index_task_count': 'datacoord.index.task_count', + 'milvus_datacoord_segment_num': 'datacoord.segment_num', + 'milvus_datacoord_stored_binlog_size': 'datacoord.stored.binlog_size', + 'milvus_datacoord_stored_index_files_size': 'datacoord.stored.index_files_size', + 'milvus_datacoord_stored_rows_num': 'datacoord.stored.rows_num', + 'milvus_datacoord_task_execute_max_latency': 'datacoord.task_execute_max_latency', + 'milvus_datacoord_watched_dml_chanel_num': 'datacoord.watched_dml_chanel_num', + 'milvus_datanode_autoflush_buffer_op_count': 'datanode.autoflush_buffer_op_count', + 'milvus_datanode_consume_bytes_count': 'datanode.consume.bytes_count', + 'milvus_datanode_consume_msg_count': 'datanode.consume.msg_count', + 'milvus_datanode_consume_tt_lag_ms': 'datanode.consume.tt_lag_ms', + 'milvus_datanode_encode_buffer_latency': 'datanode.encode_buffer_latency', + 'milvus_datanode_flowgraph_num': 'datanode.flowgraph_num', + 'milvus_datanode_flush_buffer_op_count': 'datanode.flush.buffer_op_count', + 'milvus_datanode_flush_req_count': 'datanode.flush.req_count', + 'milvus_datanode_flushed_data_rows': 'datanode.flushed_data.rows', + 'milvus_datanode_flushed_data_size': 'datanode.flushed_data.size', + 'milvus_datanode_msg_dispatcher_tt_lag_ms': 'datanode.msg.dispatcher_tt_lag_ms', + 'milvus_datanode_msg_rows_count': 'datanode.msg.rows_count', + 'milvus_datanode_save_latency': 'datanode.save_latency', + 'milvus_flushed_segment_file_num': 'flushed_segment_file_num', + 'milvus_indexnode_build_index_latency': 'indexnode.build_index_latency', + 'milvus_indexnode_encode_index_latency': 'indexnode.encode_index_latency', + 'milvus_indexnode_index_task_count': 'indexnode.index.task_count', + 'milvus_indexnode_index_task_latency_in_queue': 'indexnode.index.task_latency_in_queue', + 'milvus_indexnode_knowhere_build_index_latency': 'indexnode.knowhere_build_index_latency', + 'milvus_indexnode_save_index_latency': 'indexnode.save_index_latency', + 'milvus_meta_kv_size': 'meta.kv_size', + 'milvus_meta_op_count': 'meta.op_count', + 'milvus_meta_request_latency': 'meta.request_latency', + 'milvus_msg_queue_consumer_num': 'msg_queue_consumer_num', + 'milvus_msgstream_op_count': 'msgstream.op_count', + 'milvus_msgstream_request_latency': 'msgstream.request_latency', + 'milvus_num_node': 'num_node', + 'milvus_proxy_apply_pk_latency': 'proxy.apply.pk_latency', + 'milvus_proxy_apply_timestamp_latency': 'proxy.apply.timestamp_latency', + 'milvus_proxy_assign_segmentID_latency': 'proxy.assign_segmentID_latency', + 'milvus_proxy_cache_hit_count': 'proxy.cache.hit_count', + 'milvus_proxy_cache_update_latency': 'proxy.cache.update_latency', + 'milvus_proxy_delete_vectors_count': 'proxy.delete_vectors_count', + 'milvus_proxy_msgstream_obj_num': 'proxy.msgstream_obj_num', + 'milvus_proxy_mutation_send_latency': 'proxy.mutation_send_latency', + 'milvus_proxy_rate_limit_req_count': 'proxy.rate_limit_req_count', + 'milvus_proxy_report_value': 'proxy.report_value', + 'milvus_proxy_req_count': 'proxy.req.count', + 'milvus_proxy_req_in_queue_latency': 'proxy.req.in_queue_latency', + 'milvus_proxy_req_latency': 'proxy.req.latency', + 'milvus_proxy_send_bytes_count': 'proxy.send_bytes_count', + 'milvus_proxy_sq_decode_result_latency': 'proxy.sq.decode_result_latency', + 'milvus_proxy_sq_reduce_result_latency': 'proxy.sq.reduce_result_latency', + 'milvus_proxy_sq_wait_result_latency': 'proxy.sq.wait_result_latency', + 'milvus_proxy_sync_segment_request_length': 'proxy.sync_segment_request_length', + 'milvus_proxy_tt_lag_ms': 'proxy.tt_lag_ms', + 'milvus_querycoord_collection_num': 'querycoord.collection_num', + 'milvus_querycoord_current_target_checkpoint_unix_seconds': 'querycoord.current_target_checkpoint_unix_seconds', + 'milvus_querycoord_load_latency': 'querycoord.load.latency', + 'milvus_querycoord_load_req_count': 'querycoord.load.req_count', + 'milvus_querycoord_partition_num': 'querycoord.partition_num', + 'milvus_querycoord_querynode_num': 'querycoord.querynode_num', + 'milvus_querycoord_release_latency': 'querycoord.release.latency', + 'milvus_querycoord_release_req_count': 'querycoord.release.req_count', + 'milvus_querycoord_task_num': 'querycoord_task_num', + 'milvus_querynode_apply_bf_latency': 'querynode.apply_bf_latency', + 'milvus_querynode_collection_num': 'querynode.collection_num', + 'milvus_querynode_consume_bytes_counter': 'querynode.consume.bytes_counter', + 'milvus_querynode_consume_msg_count': 'querynode.consume.msg_count', + 'milvus_querynode_consume_tt_lag_ms': 'querynode.consume.tt_lag_ms', + 'milvus_querynode_disk_cache_evict_bytes': 'querynode.disk.cache.evict.bytes', + 'milvus_querynode_disk_cache_evict_duration': 'querynode.disk.cache.evict.duration', + 'milvus_querynode_disk_cache_evict_global_duration': 'querynode.disk.cache.evict.global_duration', + 'milvus_querynode_disk_cache_evict_total': 'querynode.disk.cache.evict.total', + 'milvus_querynode_disk_cache_load_bytes': 'querynode.disk.cache.load.bytes', + 'milvus_querynode_disk_cache_load_duration': 'querynode.disk.cache.load.duration', + 'milvus_querynode_disk_cache_load_global_duration': 'querynode.disk.cache.load.global_duration', + 'milvus_querynode_disk_cache_load_total': 'querynode.disk.cache.load.total', + 'milvus_querynode_disk_used_size': 'querynode.disk.used_size', + 'milvus_querynode_dml_vchannel_num': 'querynode.dml_vchannel_num', + 'milvus_querynode_entity_num': 'querynode.entity.num', + 'milvus_querynode_entity_size': 'querynode.entity.size', + 'milvus_querynode_execute_bytes_counter': 'querynode.execute_bytes_counter', + 'milvus_querynode_flowgraph_num': 'querynode.flowgraph_num', + 'milvus_querynode_forward_delete_latency': 'querynode.forward_delete_latency', + 'milvus_querynode_load_index_latency': 'querynode.load.index_latency', + 'milvus_querynode_load_segment_concurrency': 'querynode.load.segment.concurrency', + 'milvus_querynode_load_segment_latency': 'querynode.load.segment.latency', + 'milvus_querynode_msg_dispatcher_tt_lag_ms': 'querynode.msg_dispatcher_tt_lag_ms', + 'milvus_querynode_partition_num': 'querynode.partition_num', + 'milvus_querynode_process_insert_or_delete_latency': 'querynode.process_insert_or_delete_latency', + 'milvus_querynode_read_task_concurrency': 'querynode.read_task.concurrency', + 'milvus_querynode_read_task_ready_len': 'querynode.read_task.ready_len', + 'milvus_querynode_read_task_unsolved_len': 'querynode.read_task.unsolved_len', + 'milvus_querynode_search_group_nq': 'querynode.search.group.nq', + 'milvus_querynode_search_group_size': 'querynode.search.group.size', + 'milvus_querynode_search_group_topk': 'querynode.search.group.topk', + 'milvus_querynode_search_nq': 'querynode.search.nq', + 'milvus_querynode_search_topk': 'querynode.search.topk', + 'milvus_querynode_segment_access_duration': 'querynode.segment.access.duration', + 'milvus_querynode_segment_access_global_duration': 'querynode.segment.access.global_duration', + 'milvus_querynode_segment_access_total': 'querynode.segment.access.total', + 'milvus_querynode_segment_access_wait_cache_duration': 'querynode.segment.access.wait_cache.duration', + 'milvus_querynode_segment_access_wait_cache_global_duration': 'querynode.segment.access.wait_cache.global_duration', + 'milvus_querynode_segment_access_wait_cache_total': 'querynode.segment.access.wait_cache.total', + 'milvus_querynode_segment_latency_per_vector': 'querynode.segment.latency_per_vector', + 'milvus_querynode_segment_num': 'querynode.segment.num', + 'milvus_querynode_sq_core_latency': 'querynode.sq.core_latency', + 'milvus_querynode_sq_queue_latency': 'querynode.sq.queue.latency', + 'milvus_querynode_sq_queue_user_latency': 'querynode.sq.queue.user_latency', + 'milvus_querynode_sq_reduce_latency': 'querynode.sq.reduce_latency', + 'milvus_querynode_sq_req_count': 'querynode.sq.req.count', + 'milvus_querynode_sq_req_latency': 'querynode.sq.req.latency', + 'milvus_querynode_sq_segment_latency': 'querynode.sq.segment_latency', + 'milvus_querynode_sq_wait_tsafe_latency': 'querynode.sq.wait_tsafe_latency', + 'milvus_querynode_wait_processing_msg_count': 'querynode.wait_processing_msg_count', + 'milvus_querynode_watch_dml_channel_latency': 'querynode.watch_dml_channel_latency', + 'milvus_rootcoord_collection_num': 'rootcoord.collection_num', + 'milvus_rootcoord_credential_num': 'rootcoord.credential_num', + 'milvus_rootcoord_ddl_req_count': 'rootcoord.ddl_req.count', + 'milvus_rootcoord_ddl_req_latency': 'rootcoord.ddl_req.latency', + 'milvus_rootcoord_ddl_req_latency_in_queue': 'rootcoord.ddl_req.latency_in_queue', + 'milvus_rootcoord_dml_channel_num': 'rootcoord.dml_channel_num', + 'milvus_rootcoord_entity_num': 'rootcoord.entity_num', + 'milvus_rootcoord_force_deny_writing_counter': 'rootcoord.force_deny_writing_counter', + 'milvus_rootcoord_id_alloc_count': 'rootcoord.id_alloc_count', + 'milvus_rootcoord_indexed_entity_num': 'rootcoord.indexed_entity_num', + 'milvus_rootcoord_msgstream_obj_num': 'rootcoord.msgstream_obj_num', + 'milvus_rootcoord_num_of_roles': 'rootcoord.num_of_roles', + 'milvus_rootcoord_partition_num': 'rootcoord.partition_num', + 'milvus_rootcoord_produce_tt_lag_ms': 'rootcoord.produce_tt_lag_ms', + 'milvus_rootcoord_proxy_num': 'rootcoord.proxy_num', + 'milvus_rootcoord_qn_mem_high_water_level': 'rootcoord.qn_mem_high_water_level', + 'milvus_rootcoord_sync_timetick_latency': 'rootcoord.sync_timetick_latency', + 'milvus_rootcoord_timestamp': 'rootcoord.timestamp', + 'milvus_rootcoord_timestamp_saved': 'rootcoord.timestamp_saved', + 'milvus_runtime_info': 'runtime_info', + 'milvus_storage_kv_size': 'storage.kv_size', + 'milvus_storage_op_count': 'storage.op_count', + 'milvus_storage_request_latency': 'storage.request_latency', +} + +RENAME_LABELS_MAP = { + "version": "milvus_version", +} diff --git a/milvus/metadata.csv b/milvus/metadata.csv index c21b8ddae17a7..fa87796c0c779 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,150 +1,200 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, -milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus.cgo.cgo_duration_seconds,gauge,,second,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds,gauge,,second,,Duration of cgo call in queue.,0,milvus,, -milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, -milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, -milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, -milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, -milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, -milvus.datacoord.index.req_count,count,,,,number of building index requests,0,milvus,, -milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, -milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, -milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, -milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,, -milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, -milvus.datacoord.task_execute_max_latency,gauge,,millisecond,,latency of task execute operation,0,milvus,, -milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, -milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, -milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, -milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, -milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.datanode.encode_buffer_latency,gauge,,millisecond,,latency of encode buffer data,0,milvus,, -milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, -milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, -milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milvus,, -milvus.datanode.flushed_data.size,count,,byte,,byte size of data flushed to storage,0,milvus,, -milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus.datanode.save_latency,gauge,,millisecond,,latency of saving flush data to storage,0,milvus,, -milvus.flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, -milvus.indexnode.build_index_latency,gauge,,millisecond,,latency of build index for segment,0,milvus,, -milvus.indexnode.encode_index_latency,gauge,,millisecond,,latency of encoding the index file,0,milvus,, -milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, -milvus.indexnode.index.task_latency_in_queue,gauge,,millisecond,,latency of index task in queue,0,milvus,, -milvus.indexnode.knowhere_build_index_latency,gauge,,millisecond,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.save_index_latency,gauge,,millisecond,,latency of saving the index file,0,milvus,, -milvus.meta.kv_size,gauge,,,,kv size stats,0,milvus,, -milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, -milvus.meta.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, -milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, -milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, -milvus.msgstream.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, -milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus.proxy.apply.pk_latency,gauge,,millisecond,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.timestamp_latency,gauge,,millisecond,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.assign_segmentID_latency,gauge,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, -milvus.proxy.cache.update_latency,gauge,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, -milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, -milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus.proxy.mutation_send_latency,gauge,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, -milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.req.in_queue_latency,gauge,,millisecond,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.latency,gauge,,millisecond,,latency of each request,0,milvus,, -milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus.proxy.sq.decode_result_latency,gauge,,millisecond,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency,gauge,,millisecond,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.wait_result_latency,gauge,,millisecond,,latency that proxy waits for the result,0,milvus,, -milvus.proxy.sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, -milvus.querycoord.load.latency,gauge,,millisecond,,latency of load the entire collection,0,milvus,, -milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, -milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus.querycoord.release.latency,gauge,,millisecond,,latency of release request,0,milvus,, -milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, -milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus.querynode.apply_bf_latency,gauge,,millisecond,,apply bf cost in ms,0,milvus,, -milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, -milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, -milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, -milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.querynode.disk.cache.evict.bytes,count,,byte,,number of bytes evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.duration,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration,gauge,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.bytes,count,,byte,,number of bytes loaded from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.duration,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration,gauge,,millisecond,,global duration of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, -milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, -milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, -milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, -milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, -milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, -milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.querynode.forward_delete_latency,gauge,,millisecond,,forward delete cost in ms,0,milvus,, -milvus.querynode.load.index_latency,gauge,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus.querynode.load.segment.latency,gauge,,millisecond,,latency of load per segment,0,milvus,, -milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus.querynode.process_insert_or_delete_latency,gauge,,millisecond,,process insert or delete cost in ms,0,milvus,, -milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, -milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, -milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, -milvus.querynode.search.group.nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, -milvus.querynode.search.group.size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.querynode.search.group.topk,gauge,,,,the topK of each grouped search task,0,milvus,, -milvus.querynode.search.nq,gauge,,,,the number of queries of each search task,0,milvus,, -milvus.querynode.search.topk,gauge,,,,the top of each search task,0,milvus,, -milvus.querynode.segment.access.duration,count,,millisecond,,total time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration,gauge,,millisecond,,global time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, -milvus.querynode.segment.access.wait_cache.duration,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration,gauge,,millisecond,,global time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, -milvus.querynode.segment.latency_per_vector,gauge,,millisecond,,one vector's search latency per segment,0,milvus,, -milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus.querynode.sq.core_latency,gauge,,millisecond,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.queue.latency,gauge,,millisecond,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency,gauge,,millisecond,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.reduce_latency,gauge,,millisecond,,latency of reduce search or query result,0,milvus,, -milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, -milvus.querynode.sq.req.latency,gauge,,millisecond,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.segment_latency,gauge,,millisecond,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency,gauge,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, -milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, -milvus.querynode.watch_dml_channel_latency,gauge,,millisecond,,latency of watch dml channel,0,milvus,, -milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, -milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency,gauge,,millisecond,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue,gauge,,millisecond,,latency of each DDL operations in queue,0,milvus,, -milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, -milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, -milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, -milvus.rootcoord.id_alloc_count,count,,,,count of ID allocated,0,milvus,, -milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, -milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, -milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, -milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, -milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus.rootcoord.sync_timetick_latency,gauge,,millisecond,,latency of synchronizing timetick message,0,milvus,, -milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,, -milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, -milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, -milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, -milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, -milvus.storage.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, +milvus.ann_iterator_init_latency,gauge,,,,ann iterator init latency (ms),0,milvus,, +milvus.bf_search_cnt,gauge,,,,number of bf search per request,0,milvus,, +milvus.bitset_ratio,gauge,,,,bitset ratio,0,milvus,, +milvus.build_latency,gauge,,,,index build latency (s),0,milvus,, +milvus.cache_hit_cnt,gauge,,,,cache hit cnt per request,0,milvus,, +milvus.diskann_bitset_ratio,gauge,,,,DISKANN bitset ratio for search and range search,0,milvus,, +milvus.diskann_range_search_iters,gauge,,,,DISKANN range search iterations,0,milvus,, +milvus.diskann_search_hops,gauge,,,,DISKANN search hops,0,milvus,, +milvus.exec_latency,gauge,,,,execute latency per request,0,milvus,, +milvus.filter_connectivity_ratio,gauge,,,,avg connectivity ratio set under filtering per request,0,milvus,, +milvus.filter_mv_activated_fields_cnt,gauge,,,,avg mv activated fields per request,0,milvus,, +milvus.filter_mv_change_base_cnt,gauge,,,,mv change base cnt per request,0,milvus,, +milvus.filter_mv_only_cnt,gauge,,,,mv only cnt per request,0,milvus,, +milvus.filter_mv_supplement_ep_bool_cnt,gauge,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, +milvus.go_gc_duration_seconds,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go_goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, +milvus.go_info,gauge,,,,Information about the Go environment.,0,milvus,, +milvus.go_memstats_alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, +milvus.go_memstats_alloc_bytes_total,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, +milvus.go_memstats_buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,, +milvus.go_memstats_frees_total,count,,,,Total number of frees.,0,milvus,, +milvus.go_memstats_gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,, +milvus.go_memstats_heap_alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,, +milvus.go_memstats_heap_idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,, +milvus.go_memstats_heap_inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,, +milvus.go_memstats_heap_objects,gauge,,,,Number of allocated objects.,0,milvus,, +milvus.go_memstats_heap_released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,, +milvus.go_memstats_heap_sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,, +milvus.go_memstats_last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,, +milvus.go_memstats_lookups_total,count,,,,Total number of pointer lookups.,0,milvus,, +milvus.go_memstats_mallocs_total,count,,,,Total number of mallocs.,0,milvus,, +milvus.go_memstats_mcache_inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,, +milvus.go_memstats_mcache_sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,, +milvus.go_memstats_mspan_inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,, +milvus.go_memstats_mspan_sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,, +milvus.go_memstats_next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,, +milvus.go_memstats_other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,, +milvus.go_memstats_stack_inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,, +milvus.go_memstats_stack_sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,, +milvus.go_memstats_sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,, +milvus.go_threads,gauge,,,,Number of OS threads created.,0,milvus,, +milvus.graph_search_cnt,gauge,,,,number of graph search per request,0,milvus,, +milvus.hnsw_bitset_ratio,gauge,,,,HNSW bitset ratio for search and range search,0,milvus,, +milvus.hnsw_search_hops,gauge,,,,HNSW search hops in layer 0,0,milvus,, +milvus.internal_core_search_latency,gauge,,,,[cpp]latency(us) of search on segment,0,milvus,, +milvus.internal_mmap_allocated_space_bytes,gauge,,,,[cpp]mmap allocated space stats,0,milvus,, +milvus.internal_mmap_in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,, +milvus.internal_storage_kv_size,gauge,,,,[cpp]kv size stats,0,milvus,, +milvus.internal_storage_load_duration,gauge,,,,[cpp]durations of load segment,0,milvus,, +milvus.internal_storage_op_count,count,,,,[cpp]count of persistent data operation,0,milvus,, +milvus.internal_storage_request_latency,gauge,,,,[cpp]request latency(ms) on the client side,0,milvus,, +milvus.io_cnt,gauge,,,,io cnt per request,0,milvus,, +milvus.ivf_search_cnt,gauge,,,,number of ivf search per request,0,milvus,, +milvus.load_latency,gauge,,,,index load latency (ms),0,milvus,, +milvus.milvus_build_info,gauge,,,,Build information of milvus,0,milvus,, +milvus.milvus_cgo_active_future_total,gauge,,,,Total number of active futures.,0,milvus,, +milvus.milvus_cgo_cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.milvus_cgo_cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, +milvus.milvus_cgo_running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, +milvus.milvus_datacoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus.milvus_datacoord_consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.milvus_datacoord_datanode_num,gauge,,,,number of data nodes,0,milvus,, +milvus.milvus_datacoord_import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, +milvus.milvus_datacoord_index_node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, +milvus.milvus_datacoord_index_req_count,count,,,,number of building index requests,0,milvus,, +milvus.milvus_datacoord_segment_num,gauge,,,,number of segments,0,milvus,, +milvus.milvus_datacoord_task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, +milvus.milvus_datanode_autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, +milvus.milvus_datanode_consume_bytes_count,count,,,,,0,milvus,, +milvus.milvus_datanode_encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, +milvus.milvus_datanode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.milvus_datanode_flush_buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, +milvus.milvus_datanode_flush_req_count,count,,,,count of flush request,0,milvus,, +milvus.milvus_datanode_flushed_data_rows,count,,,,num of rows flushed to storage,0,milvus,, +milvus.milvus_datanode_flushed_data_size,count,,,,byte size of data flushed to storage,0,milvus,, +milvus.milvus_datanode_msg_rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, +milvus.milvus_datanode_save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, +milvus.milvus_flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, +milvus.milvus_indexnode_build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, +milvus.milvus_indexnode_encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, +milvus.milvus_indexnode_index_task_count,count,,,,number of tasks that index node received,0,milvus,, +milvus.milvus_indexnode_index_task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, +milvus.milvus_indexnode_knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, +milvus.milvus_indexnode_save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, +milvus.milvus_meta_kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.milvus_meta_op_count,count,,,,count of meta operation,0,milvus,, +milvus.milvus_meta_request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.milvus_msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, +milvus.milvus_msgstream_op_count,count,,,,count of stream message operation,0,milvus,, +milvus.milvus_msgstream_request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.milvus_num_node,gauge,,,,number of nodes and coordinates,0,milvus,, +milvus.milvus_proxy_apply_pk_latency,gauge,,,,latency that apply primary key,0,milvus,, +milvus.milvus_proxy_apply_timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, +milvus.milvus_proxy_assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.milvus_proxy_cache_hit_count,count,,,,count of cache hits/miss,0,milvus,, +milvus.milvus_proxy_cache_update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, +milvus.milvus_proxy_delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, +milvus.milvus_proxy_msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, +milvus.milvus_proxy_mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.milvus_proxy_rate_limit_req_count,count,,,,count of operation executed,0,milvus,, +milvus.milvus_proxy_report_value,count,,,,report value about the request,0,milvus,, +milvus.milvus_proxy_req_count,count,,,,count of operation executed,0,milvus,, +milvus.milvus_proxy_req_in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, +milvus.milvus_proxy_req_latency,gauge,,,,latency of each request,0,milvus,, +milvus.milvus_proxy_send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, +milvus.milvus_proxy_sq_decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, +milvus.milvus_proxy_sq_reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, +milvus.milvus_proxy_sq_wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, +milvus.milvus_proxy_sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.milvus_proxy_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.milvus_querycoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus.milvus_querycoord_load_latency,gauge,,,,latency of load the entire collection,0,milvus,, +milvus.milvus_querycoord_load_req_count,count,,,,count of load request,0,milvus,, +milvus.milvus_querycoord_partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.milvus_querycoord_querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, +milvus.milvus_querycoord_release_latency,gauge,,,,latency of release request,0,milvus,, +milvus.milvus_querycoord_release_req_count,count,,,,count of release request,0,milvus,, +milvus.milvus_querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, +milvus.milvus_querynode_apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, +milvus.milvus_querynode_collection_num,gauge,,,,number of collections loaded,0,milvus,, +milvus.milvus_querynode_consume_bytes_counter,count,,,,,0,milvus,, +milvus.milvus_querynode_disk_cache_evict_bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_evict_duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_evict_global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_evict_total,count,,,,number of segments evicted from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_load_bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_load_duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_load_global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, +milvus.milvus_querynode_disk_cache_load_total,count,,,,number of segments loaded from disk cache,0,milvus,, +milvus.milvus_querynode_disk_used_size,gauge,,,,disk used size(MB),0,milvus,, +milvus.milvus_querynode_dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, +milvus.milvus_querynode_execute_bytes_counter,count,,,,,0,milvus,, +milvus.milvus_querynode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.milvus_querynode_forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, +milvus.milvus_querynode_load_index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.milvus_querynode_load_segment_concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, +milvus.milvus_querynode_load_segment_latency,gauge,,,,latency of load per segment,0,milvus,, +milvus.milvus_querynode_partition_num,gauge,,,,number of partitions loaded,0,milvus,, +milvus.milvus_querynode_process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, +milvus.milvus_querynode_read_task_concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, +milvus.milvus_querynode_read_task_ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, +milvus.milvus_querynode_read_task_unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, +milvus.milvus_querynode_search_group_nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, +milvus.milvus_querynode_search_group_size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.milvus_querynode_search_group_topk,gauge,,,,the topK of each grouped search task,0,milvus,, +milvus.milvus_querynode_search_nq,gauge,,,,the number of queries of each search task,0,milvus,, +milvus.milvus_querynode_search_topk,gauge,,,,the top of each search task,0,milvus,, +milvus.milvus_querynode_segment_access_duration,count,,,,total time cost of accessing segments,0,milvus,, +milvus.milvus_querynode_segment_access_global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, +milvus.milvus_querynode_segment_access_total,count,,,,number of segments accessed,0,milvus,, +milvus.milvus_querynode_segment_access_wait_cache_duration,count,,,,total time cost of waiting for loading access,0,milvus,, +milvus.milvus_querynode_segment_access_wait_cache_global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, +milvus.milvus_querynode_segment_access_wait_cache_total,count,,,,number of segments waiting for loading access,0,milvus,, +milvus.milvus_querynode_segment_latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, +milvus.milvus_querynode_segment_num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, +milvus.milvus_querynode_sq_core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, +milvus.milvus_querynode_sq_queue_latency,gauge,,,,latency of search or query in queue,0,milvus,, +milvus.milvus_querynode_sq_queue_user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, +milvus.milvus_querynode_sq_reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, +milvus.milvus_querynode_sq_req_count,count,,,,count of search / query request,0,milvus,, +milvus.milvus_querynode_sq_req_latency,gauge,,,,latency of Search or query requests,0,milvus,, +milvus.milvus_querynode_sq_segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, +milvus.milvus_querynode_sq_wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, +milvus.milvus_querynode_wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, +milvus.milvus_querynode_watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, +milvus.milvus_rootcoord_collection_num,gauge,,,,number of collections,0,milvus,, +milvus.milvus_rootcoord_credential_num,gauge,,,,number of credentials,0,milvus,, +milvus.milvus_rootcoord_ddl_req_count,count,,,,count of DDL operations,0,milvus,, +milvus.milvus_rootcoord_ddl_req_latency,gauge,,,,latency of each DDL operations,0,milvus,, +milvus.milvus_rootcoord_ddl_req_latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, +milvus.milvus_rootcoord_dml_channel_num,gauge,,,,number of DML channels,0,milvus,, +milvus.milvus_rootcoord_entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, +milvus.milvus_rootcoord_force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, +milvus.milvus_rootcoord_id_alloc_count,count,,,,count of ID allocated,0,milvus,, +milvus.milvus_rootcoord_indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, +milvus.milvus_rootcoord_msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, +milvus.milvus_rootcoord_num_of_roles,gauge,,,,The number of roles,0,milvus,, +milvus.milvus_rootcoord_partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.milvus_rootcoord_produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, +milvus.milvus_rootcoord_proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, +milvus.milvus_rootcoord_qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, +milvus.milvus_rootcoord_sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, +milvus.milvus_rootcoord_timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, +milvus.milvus_rootcoord_timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, +milvus.milvus_runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, +milvus.process_cpu_seconds_total,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, +milvus.process_max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, +milvus.process_open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, +milvus.process_resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,, +milvus.process_start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,, +milvus.process_virtual_memory_bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,, +milvus.process_virtual_memory_max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,, +milvus.quant_compute_cnt,gauge,,,,quant compute cnt per request,0,milvus,, +milvus.queue_latency,gauge,,,,queue latency per request,0,milvus,, +milvus.range_search_latency,gauge,,,,range search latency (ms),0,milvus,, +milvus.raw_compute_cnt,gauge,,,,raw compute cnt per request,0,milvus,, +milvus.re_search_cnt,gauge,,,,number of fallback search per request,0,milvus,, +milvus.search_latency,gauge,,,,search latency (ms),0,milvus,, +milvus.search_topk,gauge,,,,search topk,0,milvus,, diff --git a/milvus/tests/common.py b/milvus/tests/common.py new file mode 100644 index 0000000000000..e6726ba18a381 --- /dev/null +++ b/milvus/tests/common.py @@ -0,0 +1,178 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from datadog_checks.dev import get_docker_hostname, get_here + +HERE = get_here() +HOST = get_docker_hostname() +PORT = 9091 + + +def get_fixture_path(filename): + return os.path.join(HERE, 'fixtures', filename) + + +MOCKED_INSTANCE = { + "openmetrics_endpoint": f"http://{HOST}:{PORT}/metrics", + 'tags': ['test:tag'], +} + +COMPOSE_FILE = os.path.join(HERE, 'docker', 'docker-compose.yaml') + +TEST_METRICS = { + 'milvus.build_info': 'gauge', + 'milvus.cgo.active_future_total': 'gauge', + 'milvus.cgo.cgo_duration_seconds': 'gauge', + 'milvus.cgo.cgo_queue_duration_seconds': 'gauge', + 'milvus.cgo.running_cgo_call_total': 'gauge', + 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', + 'milvus.datacoord.collection_num': 'gauge', + 'milvus.datacoord.consume_datanode_tt_lag_ms': 'gauge', + 'milvus.datacoord.datanode_num': 'gauge', + 'milvus.datacoord.import_tasks': 'gauge', + 'milvus.datacoord.index.node_num': 'gauge', + 'milvus.datacoord.index.req_count': 'count', + 'milvus.datacoord.index.task_count': 'gauge', + 'milvus.datacoord.segment_num': 'gauge', + 'milvus.datacoord.stored.binlog_size': 'gauge', + 'milvus.datacoord.stored.index_files_size': 'gauge', + 'milvus.datacoord.stored.rows_num': 'gauge', + 'milvus.datacoord.task_execute_max_latency': 'gauge', + 'milvus.datacoord.watched_dml_chanel_num': 'gauge', + 'milvus.datanode.autoflush_buffer_op_count': 'count', + 'milvus.datanode.consume.bytes_count': 'count', + 'milvus.datanode.consume.msg_count': 'count', + 'milvus.datanode.consume.tt_lag_ms': 'gauge', + 'milvus.datanode.encode_buffer_latency': 'gauge', + 'milvus.datanode.flowgraph_num': 'gauge', + 'milvus.datanode.flush.buffer_op_count': 'count', + 'milvus.datanode.flush.req_count': 'count', + 'milvus.datanode.flushed_data.rows': 'count', + 'milvus.datanode.flushed_data.size': 'count', + 'milvus.datanode.msg.dispatcher_tt_lag_ms': 'gauge', + 'milvus.datanode.msg.rows_count': 'count', + 'milvus.datanode.save_latency': 'gauge', + 'milvus.flushed_segment_file_num': 'gauge', + 'milvus.indexnode.build_index_latency': 'gauge', + 'milvus.indexnode.encode_index_latency': 'gauge', + 'milvus.indexnode.index.task_count': 'count', + 'milvus.indexnode.index.task_latency_in_queue': 'gauge', + 'milvus.indexnode.knowhere_build_index_latency': 'gauge', + 'milvus.indexnode.save_index_latency': 'gauge', + 'milvus.meta.kv_size': 'gauge', + 'milvus.meta.op_count': 'count', + 'milvus.meta.request_latency': 'gauge', + 'milvus.msg_queue_consumer_num': 'gauge', + 'milvus.msgstream.op_count': 'count', + 'milvus.msgstream.request_latency': 'gauge', + 'milvus.num_node': 'gauge', + 'milvus.proxy.apply.pk_latency': 'gauge', + 'milvus.proxy.apply.timestamp_latency': 'gauge', + 'milvus.proxy.assign_segmentID_latency': 'gauge', + 'milvus.proxy.cache.hit_count': 'count', + 'milvus.proxy.cache.update_latency': 'gauge', + 'milvus.proxy.delete_vectors_count': 'count', + 'milvus.proxy.msgstream_obj_num': 'gauge', + 'milvus.proxy.mutation_send_latency': 'gauge', + 'milvus.proxy.rate_limit_req_count': 'count', + 'milvus.proxy.report_value': 'count', + 'milvus.proxy.req.count': 'count', + 'milvus.proxy.req.in_queue_latency': 'gauge', + 'milvus.proxy.req.latency': 'gauge', + 'milvus.proxy.send_bytes_count': 'count', + 'milvus.proxy.sq.decode_result_latency': 'gauge', + 'milvus.proxy.sq.reduce_result_latency': 'gauge', + 'milvus.proxy.sq.wait_result_latency': 'gauge', + 'milvus.proxy.sync_segment_request_length': 'gauge', + 'milvus.proxy.tt_lag_ms': 'gauge', + 'milvus.querycoord.collection_num': 'gauge', + 'milvus.querycoord.current_target_checkpoint_unix_seconds': 'gauge', + 'milvus.querycoord.load.latency': 'gauge', + 'milvus.querycoord.load.req_count': 'count', + 'milvus.querycoord.partition_num': 'gauge', + 'milvus.querycoord.querynode_num': 'gauge', + 'milvus.querycoord.release.latency': 'gauge', + 'milvus.querycoord.release.req_count': 'count', + 'milvus.querycoord_task_num': 'gauge', + 'milvus.querynode.apply_bf_latency': 'gauge', + 'milvus.querynode.collection_num': 'gauge', + 'milvus.querynode.consume.bytes_counter': 'count', + 'milvus.querynode.consume.msg_count': 'count', + 'milvus.querynode.consume.tt_lag_ms': 'gauge', + 'milvus.querynode.disk.cache.evict.bytes': 'count', + 'milvus.querynode.disk.cache.evict.duration': 'count', + 'milvus.querynode.disk.cache.evict.global_duration': 'gauge', + 'milvus.querynode.disk.cache.evict.total': 'count', + 'milvus.querynode.disk.cache.load.bytes': 'count', + 'milvus.querynode.disk.cache.load.duration': 'count', + 'milvus.querynode.disk.cache.load.global_duration': 'gauge', + 'milvus.querynode.disk.cache.load.total': 'count', + 'milvus.querynode.disk.used_size': 'gauge', + 'milvus.querynode.dml_vchannel_num': 'gauge', + 'milvus.querynode.entity.num': 'gauge', + 'milvus.querynode.entity.size': 'gauge', + 'milvus.querynode.execute_bytes_counter': 'count', + 'milvus.querynode.flowgraph_num': 'gauge', + 'milvus.querynode.forward_delete_latency': 'gauge', + 'milvus.querynode.load.index_latency': 'gauge', + 'milvus.querynode.load.segment.concurrency': 'gauge', + 'milvus.querynode.load.segment.latency': 'gauge', + 'milvus.querynode.msg_dispatcher_tt_lag_ms': 'gauge', + 'milvus.querynode.partition_num': 'gauge', + 'milvus.querynode.process_insert_or_delete_latency': 'gauge', + 'milvus.querynode.read_task.concurrency': 'gauge', + 'milvus.querynode.read_task.ready_len': 'gauge', + 'milvus.querynode.read_task.unsolved_len': 'gauge', + 'milvus.querynode.search.group.nq': 'gauge', + 'milvus.querynode.search.group.size': 'gauge', + 'milvus.querynode.search.group.topk': 'gauge', + 'milvus.querynode.search.nq': 'gauge', + 'milvus.querynode.search.topk': 'gauge', + 'milvus.querynode.segment.access.duration': 'count', + 'milvus.querynode.segment.access.global_duration': 'gauge', + 'milvus.querynode.segment.access.total': 'count', + 'milvus.querynode.segment.access.wait_cache.duration': 'count', + 'milvus.querynode.segment.access.wait_cache.global_duration': 'gauge', + 'milvus.querynode.segment.access.wait_cache.total': 'count', + 'milvus.querynode.segment.latency_per_vector': 'gauge', + 'milvus.querynode.segment.num': 'gauge', + 'milvus.querynode.sq.core_latency': 'gauge', + 'milvus.querynode.sq.queue.latency': 'gauge', + 'milvus.querynode.sq.queue.user_latency': 'gauge', + 'milvus.querynode.sq.reduce_latency': 'gauge', + 'milvus.querynode.sq.req.count': 'count', + 'milvus.querynode.sq.req.latency': 'gauge', + 'milvus.querynode.sq.segment_latency': 'gauge', + 'milvus.querynode.sq.wait_tsafe_latency': 'gauge', + 'milvus.querynode.wait_processing_msg_count': 'gauge', + 'milvus.querynode.watch_dml_channel_latency': 'gauge', + 'milvus.rootcoord.collection_num': 'gauge', + 'milvus.rootcoord.credential_num': 'gauge', + 'milvus.rootcoord.ddl_req.count': 'count', + 'milvus.rootcoord.ddl_req.latency': 'gauge', + 'milvus.rootcoord.ddl_req.latency_in_queue': 'gauge', + 'milvus.rootcoord.dml_channel_num': 'gauge', + 'milvus.rootcoord.entity_num': 'gauge', + 'milvus.rootcoord.force_deny_writing_counter': 'count', + 'milvus.rootcoord.id_alloc_count': 'count', + 'milvus.rootcoord.indexed_entity_num': 'gauge', + 'milvus.rootcoord.msgstream_obj_num': 'gauge', + 'milvus.rootcoord.num_of_roles': 'gauge', + 'milvus.rootcoord.partition_num': 'gauge', + 'milvus.rootcoord.produce_tt_lag_ms': 'gauge', + 'milvus.rootcoord.proxy_num': 'gauge', + 'milvus.rootcoord.qn_mem_high_water_level': 'gauge', + 'milvus.rootcoord.sync_timetick_latency': 'gauge', + 'milvus.rootcoord.timestamp': 'gauge', + 'milvus.rootcoord.timestamp_saved': 'gauge', + 'milvus.runtime_info': 'gauge', + 'milvus.storage.kv_size': 'gauge', + 'milvus.storage.op_count': 'count', + 'milvus.storage.request_latency': 'gauge', +} + +RENAMED_LABELS = { + "milvus.build_info": 'milvus_version:v2.3.15', +} diff --git a/milvus/tests/conftest.py b/milvus/tests/conftest.py index c8ac597a9862c..c752c4ca012ec 100644 --- a/milvus/tests/conftest.py +++ b/milvus/tests/conftest.py @@ -1,14 +1,31 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import copy +import logging + import pytest +from datadog_checks.dev import docker_run +from datadog_checks.dev.conditions import CheckDockerLogs, CheckEndpoints + +from . import common + @pytest.fixture(scope='session') def dd_environment(): - yield + compose_file = common.COMPOSE_FILE + conditions = [ + CheckDockerLogs(identifier='caddy', patterns=['server running']), + CheckEndpoints(common.MOCKED_INSTANCE["openmetrics_endpoint"]), + ] + logging.info(conditions) + with docker_run(compose_file, conditions=conditions): + yield { + 'instances': [common.MOCKED_INSTANCE], + } @pytest.fixture def instance(): - return {} + return copy.deepcopy(common.MOCKED_INSTANCE) diff --git a/milvus/tests/docker/Caddyfile b/milvus/tests/docker/Caddyfile new file mode 100644 index 0000000000000..96d00534e6cb3 --- /dev/null +++ b/milvus/tests/docker/Caddyfile @@ -0,0 +1,4 @@ +:9091 { + root * /metrics + file_server +} diff --git a/milvus/tests/docker/docker-compose.yaml b/milvus/tests/docker/docker-compose.yaml new file mode 100644 index 0000000000000..8bb27b6a4468a --- /dev/null +++ b/milvus/tests/docker/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + +# Caddy will create host a site with the metrics to exposes + caddy: + image: caddy:2.7 + container_name: caddy + ports: + - "9091:9091" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ../fixtures/milvus_payload.txt:/metrics/metrics diff --git a/milvus/tests/fixtures/docker_payload.txt b/milvus/tests/fixtures/milvus_payload.txt similarity index 100% rename from milvus/tests/fixtures/docker_payload.txt rename to milvus/tests/fixtures/milvus_payload.txt diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py index fbf965dd4ca18..52fee01ea157b 100644 --- a/milvus/tests/test_unit.py +++ b/milvus/tests/test_unit.py @@ -2,45 +2,51 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Any, Callable, Dict # noqa: F401 - import pytest -from datadog_checks.base import AgentCheck # noqa: F401 -from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 -from datadog_checks.dev.utils import get_metadata_metrics +from datadog_checks.dev.utils import get_metadata_metrics from datadog_checks.milvus import MilvusCheck +from .common import TEST_METRICS, RENAMED_LABELS, get_fixture_path + + +def test_check(dd_run_check, aggregator, instance, mock_http_response): + mock_http_response(file_path=get_fixture_path('milvus_payload.txt')) -def test_check(dd_run_check, aggregator, instance): - # type: (Callable[[AgentCheck, bool], None], AggregatorStub, Dict[str, Any]) -> None check = MilvusCheck('milvus', {}, [instance]) dd_run_check(check) + for metric, metric_type in TEST_METRICS.items(): + aggregator.assert_metric(metric, metric_type=aggregator.METRIC_ENUM_MAP[metric_type]) + aggregator.assert_metric_has_tag(metric, 'test:tag') + + for metric, tag in RENAMED_LABELS.items(): + aggregator.assert_metric_has_tag(metric, tag) + aggregator.assert_all_metrics_covered() aggregator.assert_metrics_using_metadata(get_metadata_metrics()) def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance): - # type: (Callable[[AgentCheck, bool], None], AggregatorStub, Dict[str, Any]) -> None check = MilvusCheck('milvus', {}, [instance]) dd_run_check(check) aggregator.assert_service_check('milvus.can_connect', MilvusCheck.CRITICAL) + def test_empty_instance(dd_run_check): with pytest.raises( Exception, match='InstanceConfig`:\nopenmetrics_endpoint\n Field required', ): - check = MilvusCheck('aws_neuron', {}, [{}]) + check = MilvusCheck('milvus', {}, [{}]) dd_run_check(check) def test_custom_validation(dd_run_check): - endpoint = 'aws_neuron:2112/metrics' + endpoint = 'milvus:2112/metrics' with pytest.raises( Exception, match='openmetrics_endpoint: {} is incorrectly configured'.format(endpoint), ): - check = MilvusCheck('aws_neuron', {}, [{'openmetrics_endpoint': endpoint}]) + check = MilvusCheck('milvus', {}, [{'openmetrics_endpoint': endpoint}]) dd_run_check(check) From 6da32edf92b48301644060a70ee4b3658529847b Mon Sep 17 00:00:00 2001 From: David Kirov Date: Wed, 27 Nov 2024 16:24:24 +0100 Subject: [PATCH 13/31] Fix metadata.csv --- milvus/metadata.csv | 348 +++++++++++++++++++------------------------- 1 file changed, 149 insertions(+), 199 deletions(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index fa87796c0c779..da38992ccd319 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,200 +1,150 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -milvus.ann_iterator_init_latency,gauge,,,,ann iterator init latency (ms),0,milvus,, -milvus.bf_search_cnt,gauge,,,,number of bf search per request,0,milvus,, -milvus.bitset_ratio,gauge,,,,bitset ratio,0,milvus,, -milvus.build_latency,gauge,,,,index build latency (s),0,milvus,, -milvus.cache_hit_cnt,gauge,,,,cache hit cnt per request,0,milvus,, -milvus.diskann_bitset_ratio,gauge,,,,DISKANN bitset ratio for search and range search,0,milvus,, -milvus.diskann_range_search_iters,gauge,,,,DISKANN range search iterations,0,milvus,, -milvus.diskann_search_hops,gauge,,,,DISKANN search hops,0,milvus,, -milvus.exec_latency,gauge,,,,execute latency per request,0,milvus,, -milvus.filter_connectivity_ratio,gauge,,,,avg connectivity ratio set under filtering per request,0,milvus,, -milvus.filter_mv_activated_fields_cnt,gauge,,,,avg mv activated fields per request,0,milvus,, -milvus.filter_mv_change_base_cnt,gauge,,,,mv change base cnt per request,0,milvus,, -milvus.filter_mv_only_cnt,gauge,,,,mv only cnt per request,0,milvus,, -milvus.filter_mv_supplement_ep_bool_cnt,gauge,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, -milvus.go_gc_duration_seconds,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go_goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, -milvus.go_info,gauge,,,,Information about the Go environment.,0,milvus,, -milvus.go_memstats_alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, -milvus.go_memstats_alloc_bytes_total,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, -milvus.go_memstats_buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,, -milvus.go_memstats_frees_total,count,,,,Total number of frees.,0,milvus,, -milvus.go_memstats_gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,, -milvus.go_memstats_heap_alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,, -milvus.go_memstats_heap_idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,, -milvus.go_memstats_heap_inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,, -milvus.go_memstats_heap_objects,gauge,,,,Number of allocated objects.,0,milvus,, -milvus.go_memstats_heap_released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,, -milvus.go_memstats_heap_sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,, -milvus.go_memstats_last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,, -milvus.go_memstats_lookups_total,count,,,,Total number of pointer lookups.,0,milvus,, -milvus.go_memstats_mallocs_total,count,,,,Total number of mallocs.,0,milvus,, -milvus.go_memstats_mcache_inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,, -milvus.go_memstats_mcache_sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,, -milvus.go_memstats_mspan_inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,, -milvus.go_memstats_mspan_sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,, -milvus.go_memstats_next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,, -milvus.go_memstats_other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,, -milvus.go_memstats_stack_inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,, -milvus.go_memstats_stack_sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,, -milvus.go_memstats_sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,, -milvus.go_threads,gauge,,,,Number of OS threads created.,0,milvus,, -milvus.graph_search_cnt,gauge,,,,number of graph search per request,0,milvus,, -milvus.hnsw_bitset_ratio,gauge,,,,HNSW bitset ratio for search and range search,0,milvus,, -milvus.hnsw_search_hops,gauge,,,,HNSW search hops in layer 0,0,milvus,, -milvus.internal_core_search_latency,gauge,,,,[cpp]latency(us) of search on segment,0,milvus,, -milvus.internal_mmap_allocated_space_bytes,gauge,,,,[cpp]mmap allocated space stats,0,milvus,, -milvus.internal_mmap_in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,, -milvus.internal_storage_kv_size,gauge,,,,[cpp]kv size stats,0,milvus,, -milvus.internal_storage_load_duration,gauge,,,,[cpp]durations of load segment,0,milvus,, -milvus.internal_storage_op_count,count,,,,[cpp]count of persistent data operation,0,milvus,, -milvus.internal_storage_request_latency,gauge,,,,[cpp]request latency(ms) on the client side,0,milvus,, -milvus.io_cnt,gauge,,,,io cnt per request,0,milvus,, -milvus.ivf_search_cnt,gauge,,,,number of ivf search per request,0,milvus,, -milvus.load_latency,gauge,,,,index load latency (ms),0,milvus,, -milvus.milvus_build_info,gauge,,,,Build information of milvus,0,milvus,, -milvus.milvus_cgo_active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus.milvus_cgo_cgo_duration_seconds,gauge,,,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.milvus_cgo_cgo_queue_duration_seconds,gauge,,,,Duration of cgo call in queue.,0,milvus,, -milvus.milvus_cgo_running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, -milvus.milvus_datacoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus.milvus_datacoord_consume_datanode_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus.milvus_datacoord_datanode_num,gauge,,,,number of data nodes,0,milvus,, -milvus.milvus_datacoord_import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, -milvus.milvus_datacoord_index_node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, -milvus.milvus_datacoord_index_req_count,count,,,,number of building index requests,0,milvus,, -milvus.milvus_datacoord_segment_num,gauge,,,,number of segments,0,milvus,, -milvus.milvus_datacoord_task_execute_max_latency,gauge,,,,latency of task execute operation,0,milvus,, -milvus.milvus_datanode_autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, -milvus.milvus_datanode_consume_bytes_count,count,,,,,0,milvus,, -milvus.milvus_datanode_encode_buffer_latency,gauge,,,,latency of encode buffer data,0,milvus,, -milvus.milvus_datanode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.milvus_datanode_flush_buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, -milvus.milvus_datanode_flush_req_count,count,,,,count of flush request,0,milvus,, -milvus.milvus_datanode_flushed_data_rows,count,,,,num of rows flushed to storage,0,milvus,, -milvus.milvus_datanode_flushed_data_size,count,,,,byte size of data flushed to storage,0,milvus,, -milvus.milvus_datanode_msg_rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus.milvus_datanode_save_latency,gauge,,,,latency of saving flush data to storage,0,milvus,, -milvus.milvus_flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, -milvus.milvus_indexnode_build_index_latency,gauge,,,,latency of build index for segment,0,milvus,, -milvus.milvus_indexnode_encode_index_latency,gauge,,,,latency of encoding the index file,0,milvus,, -milvus.milvus_indexnode_index_task_count,count,,,,number of tasks that index node received,0,milvus,, -milvus.milvus_indexnode_index_task_latency_in_queue,gauge,,,,latency of index task in queue,0,milvus,, -milvus.milvus_indexnode_knowhere_build_index_latency,gauge,,,,latency of building the index by knowhere,0,milvus,, -milvus.milvus_indexnode_save_index_latency,gauge,,,,latency of saving the index file,0,milvus,, -milvus.milvus_meta_kv_size,gauge,,,,kv size stats,0,milvus,, -milvus.milvus_meta_op_count,count,,,,count of meta operation,0,milvus,, -milvus.milvus_meta_request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus.milvus_msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, -milvus.milvus_msgstream_op_count,count,,,,count of stream message operation,0,milvus,, -milvus.milvus_msgstream_request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus.milvus_num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus.milvus_proxy_apply_pk_latency,gauge,,,,latency that apply primary key,0,milvus,, -milvus.milvus_proxy_apply_timestamp_latency,gauge,,,,latency that proxy apply timestamp,0,milvus,, -milvus.milvus_proxy_assign_segmentID_latency,gauge,,,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.milvus_proxy_cache_hit_count,count,,,,count of cache hits/miss,0,milvus,, -milvus.milvus_proxy_cache_update_latency,gauge,,,,latency that proxy update cache when cache miss,0,milvus,, -milvus.milvus_proxy_delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, -milvus.milvus_proxy_msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus.milvus_proxy_mutation_send_latency,gauge,,,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.milvus_proxy_rate_limit_req_count,count,,,,count of operation executed,0,milvus,, -milvus.milvus_proxy_report_value,count,,,,report value about the request,0,milvus,, -milvus.milvus_proxy_req_count,count,,,,count of operation executed,0,milvus,, -milvus.milvus_proxy_req_in_queue_latency,gauge,,,,latency which request waits in the queue,0,milvus,, -milvus.milvus_proxy_req_latency,gauge,,,,latency of each request,0,milvus,, -milvus.milvus_proxy_send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus.milvus_proxy_sq_decode_result_latency,gauge,,,,latency that proxy decodes the search result,0,milvus,, -milvus.milvus_proxy_sq_reduce_result_latency,gauge,,,,latency that proxy reduces search result,0,milvus,, -milvus.milvus_proxy_sq_wait_result_latency,gauge,,,,latency that proxy waits for the result,0,milvus,, -milvus.milvus_proxy_sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.milvus_proxy_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus.milvus_querycoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus.milvus_querycoord_load_latency,gauge,,,,latency of load the entire collection,0,milvus,, -milvus.milvus_querycoord_load_req_count,count,,,,count of load request,0,milvus,, -milvus.milvus_querycoord_partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.milvus_querycoord_querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus.milvus_querycoord_release_latency,gauge,,,,latency of release request,0,milvus,, -milvus.milvus_querycoord_release_req_count,count,,,,count of release request,0,milvus,, -milvus.milvus_querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus.milvus_querynode_apply_bf_latency,gauge,,,,apply bf cost in ms,0,milvus,, -milvus.milvus_querynode_collection_num,gauge,,,,number of collections loaded,0,milvus,, -milvus.milvus_querynode_consume_bytes_counter,count,,,,,0,milvus,, -milvus.milvus_querynode_disk_cache_evict_bytes,count,,,,number of bytes evicted from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_evict_duration,count,,,,total time cost of evicting segments from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_evict_global_duration,gauge,,,,global duration of evicting segments from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_evict_total,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_load_bytes,count,,,,number of bytes loaded from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_load_duration,count,,,,total time cost of loading segments from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_load_global_duration,gauge,,,,global duration of loading segments from disk cache,0,milvus,, -milvus.milvus_querynode_disk_cache_load_total,count,,,,number of segments loaded from disk cache,0,milvus,, -milvus.milvus_querynode_disk_used_size,gauge,,,,disk used size(MB),0,milvus,, -milvus.milvus_querynode_dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, -milvus.milvus_querynode_execute_bytes_counter,count,,,,,0,milvus,, -milvus.milvus_querynode_flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.milvus_querynode_forward_delete_latency,gauge,,,,forward delete cost in ms,0,milvus,, -milvus.milvus_querynode_load_index_latency,gauge,,,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus.milvus_querynode_load_segment_concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus.milvus_querynode_load_segment_latency,gauge,,,,latency of load per segment,0,milvus,, -milvus.milvus_querynode_partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus.milvus_querynode_process_insert_or_delete_latency,gauge,,,,process insert or delete cost in ms,0,milvus,, -milvus.milvus_querynode_read_task_concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, -milvus.milvus_querynode_read_task_ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, -milvus.milvus_querynode_read_task_unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, -milvus.milvus_querynode_search_group_nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, -milvus.milvus_querynode_search_group_size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.milvus_querynode_search_group_topk,gauge,,,,the topK of each grouped search task,0,milvus,, -milvus.milvus_querynode_search_nq,gauge,,,,the number of queries of each search task,0,milvus,, -milvus.milvus_querynode_search_topk,gauge,,,,the top of each search task,0,milvus,, -milvus.milvus_querynode_segment_access_duration,count,,,,total time cost of accessing segments,0,milvus,, -milvus.milvus_querynode_segment_access_global_duration,gauge,,,,global time cost of accessing segments,0,milvus,, -milvus.milvus_querynode_segment_access_total,count,,,,number of segments accessed,0,milvus,, -milvus.milvus_querynode_segment_access_wait_cache_duration,count,,,,total time cost of waiting for loading access,0,milvus,, -milvus.milvus_querynode_segment_access_wait_cache_global_duration,gauge,,,,global time cost of waiting for loading access,0,milvus,, -milvus.milvus_querynode_segment_access_wait_cache_total,count,,,,number of segments waiting for loading access,0,milvus,, -milvus.milvus_querynode_segment_latency_per_vector,gauge,,,,one vector's search latency per segment,0,milvus,, -milvus.milvus_querynode_segment_num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus.milvus_querynode_sq_core_latency,gauge,,,,latency of search or query latency in segcore,0,milvus,, -milvus.milvus_querynode_sq_queue_latency,gauge,,,,latency of search or query in queue,0,milvus,, -milvus.milvus_querynode_sq_queue_user_latency,gauge,,,,latency per user of search or query in queue,0,milvus,, -milvus.milvus_querynode_sq_reduce_latency,gauge,,,,latency of reduce search or query result,0,milvus,, -milvus.milvus_querynode_sq_req_count,count,,,,count of search / query request,0,milvus,, -milvus.milvus_querynode_sq_req_latency,gauge,,,,latency of Search or query requests,0,milvus,, -milvus.milvus_querynode_sq_segment_latency,gauge,,,,latency of search or query per segment,0,milvus,, -milvus.milvus_querynode_sq_wait_tsafe_latency,gauge,,,,latency of search or query to wait for tsafe,0,milvus,, -milvus.milvus_querynode_wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, -milvus.milvus_querynode_watch_dml_channel_latency,gauge,,,,latency of watch dml channel,0,milvus,, -milvus.milvus_rootcoord_collection_num,gauge,,,,number of collections,0,milvus,, -milvus.milvus_rootcoord_credential_num,gauge,,,,number of credentials,0,milvus,, -milvus.milvus_rootcoord_ddl_req_count,count,,,,count of DDL operations,0,milvus,, -milvus.milvus_rootcoord_ddl_req_latency,gauge,,,,latency of each DDL operations,0,milvus,, -milvus.milvus_rootcoord_ddl_req_latency_in_queue,gauge,,,,latency of each DDL operations in queue,0,milvus,, -milvus.milvus_rootcoord_dml_channel_num,gauge,,,,number of DML channels,0,milvus,, -milvus.milvus_rootcoord_entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, -milvus.milvus_rootcoord_force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, -milvus.milvus_rootcoord_id_alloc_count,count,,,,count of ID allocated,0,milvus,, -milvus.milvus_rootcoord_indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, -milvus.milvus_rootcoord_msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, -milvus.milvus_rootcoord_num_of_roles,gauge,,,,The number of roles,0,milvus,, -milvus.milvus_rootcoord_partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.milvus_rootcoord_produce_tt_lag_ms,gauge,,,,now time minus tt per physical channel,0,milvus,, -milvus.milvus_rootcoord_proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, -milvus.milvus_rootcoord_qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus.milvus_rootcoord_sync_timetick_latency,gauge,,,,latency of synchronizing timetick message,0,milvus,, -milvus.milvus_rootcoord_timestamp,gauge,,,,lateste timestamp allocated in memory,0,milvus,, -milvus.milvus_rootcoord_timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, -milvus.milvus_runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, -milvus.process_cpu_seconds_total,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, -milvus.process_max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, -milvus.process_open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, -milvus.process_resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,, -milvus.process_start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,, -milvus.process_virtual_memory_bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,, -milvus.process_virtual_memory_max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,, -milvus.quant_compute_cnt,gauge,,,,quant compute cnt per request,0,milvus,, -milvus.queue_latency,gauge,,,,queue latency per request,0,milvus,, -milvus.range_search_latency,gauge,,,,range search latency (ms),0,milvus,, -milvus.raw_compute_cnt,gauge,,,,raw compute cnt per request,0,milvus,, -milvus.re_search_cnt,gauge,,,,number of fallback search per request,0,milvus,, -milvus.search_latency,gauge,,,,search latency (ms),0,milvus,, -milvus.search_topk,gauge,,,,search topk,0,milvus,, +milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, +milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, +milvus.cgo.cgo_duration_seconds,gauge,,second,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds,gauge,,second,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, +milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, +milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, +milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, +milvus.datacoord.index.req_count,count,,,,number of building index requests,0,milvus,, +milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, +milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, +milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, +milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,, +milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, +milvus.datacoord.task_execute_max_latency,gauge,,millisecond,,latency of task execute operation,0,milvus,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, +milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, +milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, +milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.datanode.encode_buffer_latency,gauge,,millisecond,,latency of encode buffer data,0,milvus,, +milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, +milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, +milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milvus,, +milvus.datanode.flushed_data.size,count,,byte,,byte size of data flushed to storage,0,milvus,, +milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, +milvus.datanode.save_latency,gauge,,millisecond,,latency of saving flush data to storage,0,milvus,, +milvus.flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, +milvus.indexnode.build_index_latency,gauge,,millisecond,,latency of build index for segment,0,milvus,, +milvus.indexnode.encode_index_latency,gauge,,millisecond,,latency of encoding the index file,0,milvus,, +milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, +milvus.indexnode.index.task_latency_in_queue,gauge,,millisecond,,latency of index task in queue,0,milvus,, +milvus.indexnode.knowhere_build_index_latency,gauge,,millisecond,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.save_index_latency,gauge,,millisecond,,latency of saving the index file,0,milvus,, +milvus.meta.kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, +milvus.meta.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, +milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, +milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, +milvus.msgstream.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, +milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, +milvus.proxy.apply.pk_latency,gauge,,millisecond,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.timestamp_latency,gauge,,millisecond,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.assign_segmentID_latency,gauge,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, +milvus.proxy.cache.update_latency,gauge,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, +milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, +milvus.proxy.mutation_send_latency,gauge,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, +milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, +milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, +milvus.proxy.req.in_queue_latency,gauge,,millisecond,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.latency,gauge,,millisecond,,latency of each request,0,milvus,, +milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, +milvus.proxy.sq.decode_result_latency,gauge,,millisecond,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency,gauge,,millisecond,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.wait_result_latency,gauge,,millisecond,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, +milvus.querycoord.load.latency,gauge,,millisecond,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, +milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, +milvus.querycoord.release.latency,gauge,,millisecond,,latency of release request,0,milvus,, +milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, +milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, +milvus.querynode.apply_bf_latency,gauge,,millisecond,,apply bf cost in ms,0,milvus,, +milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, +milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, +milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.querynode.disk.cache.evict.bytes,count,,byte,,number of bytes evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.duration,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration,gauge,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.bytes,count,,byte,,number of bytes loaded from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.duration,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration,gauge,,millisecond,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, +milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, +milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, +milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, +milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, +milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, +milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, +milvus.querynode.forward_delete_latency,gauge,,millisecond,,forward delete cost in ms,0,milvus,, +milvus.querynode.load.index_latency,gauge,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, +milvus.querynode.load.segment.latency,gauge,,millisecond,,latency of load per segment,0,milvus,, +milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, +milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, +milvus.querynode.process_insert_or_delete_latency,gauge,,millisecond,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, +milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, +milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, +milvus.querynode.search.group.nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, +milvus.querynode.search.group.size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.querynode.search.group.topk,gauge,,,,the topK of each grouped search task,0,milvus,, +milvus.querynode.search.nq,gauge,,,,the number of queries of each search task,0,milvus,, +milvus.querynode.search.topk,gauge,,,,the top of each search task,0,milvus,, +milvus.querynode.segment.access.duration,count,,millisecond,,total time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration,gauge,,millisecond,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, +milvus.querynode.segment.access.wait_cache.duration,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration,gauge,,millisecond,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, +milvus.querynode.segment.latency_per_vector,gauge,,millisecond,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, +milvus.querynode.sq.core_latency,gauge,,millisecond,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.queue.latency,gauge,,millisecond,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency,gauge,,millisecond,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.reduce_latency,gauge,,millisecond,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, +milvus.querynode.sq.req.latency,gauge,,millisecond,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.segment_latency,gauge,,millisecond,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency,gauge,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, +milvus.querynode.watch_dml_channel_latency,gauge,,millisecond,,latency of watch dml channel,0,milvus,, +milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, +milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, +milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency,gauge,,millisecond,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue,gauge,,millisecond,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, +milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, +milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, +milvus.rootcoord.id_alloc_count,count,,,,count of ID allocated,0,milvus,, +milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, +milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, +milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, +milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, +milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, +milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, +milvus.rootcoord.sync_timetick_latency,gauge,,millisecond,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,, +milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, +milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, +milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, +milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, From 66d50f88d762689fba369a6b3bfe4ff49ad0a7e7 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Wed, 27 Nov 2024 19:09:27 +0100 Subject: [PATCH 14/31] Add non-Milvus specific metrics --- milvus/metadata.csv | 67 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index da38992ccd319..8ac12317f86ad 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -148,3 +148,70 @@ milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, +milvus.bf_search_cnt,gauge,,,,number of bf search per request,0,milvus,, +milvus.bitset_ratio,gauge,,,,bitset ratio,0,milvus,, +milvus.build_latency,gauge,,,,index build latency (s),0,milvus,, +milvus.cache_hit_cnt,gauge,,,,cache hit cnt per request,0,milvus,, +milvus.diskann.range_search_iters,gauge,,,,DISKANN range search iterations,0,milvus,, +milvus.diskann.search_hops,gauge,,,,DISKANN search hops,0,milvus,, +milvus.diskann_bitset_ratio,gauge,,,,DISKANN bitset ratio for search and range search,0,milvus,, +milvus.exec_latency,gauge,,,,execute latency per request,0,milvus,, +milvus.filter.connectivity_ratio,gauge,,,,avg connectivity ratio set under filtering per request,0,milvus,, +milvus.filter.mv.activated_fields_cnt,gauge,,,,avg mv activated fields per request,0,milvus,, +milvus.filter.mv.change_base_cnt,gauge,,,,mv change base cnt per request,0,milvus,, +milvus.filter.mv.only_cnt,gauge,,,,mv only cnt per request,0,milvus,, +milvus.filter.mv.supplement_ep_bool_cnt,gauge,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, +milvus.go.gc_duration_seconds,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, +milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,, +milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, +milvus.go.memstats.alloc_bytes_total,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, +milvus.go.memstats.buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,, +milvus.go.memstats.frees_total,count,,,,Total number of frees.,0,milvus,, +milvus.go.memstats.gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,, +milvus.go.memstats.heap.alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,, +milvus.go.memstats.heap.idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,, +milvus.go.memstats.heap.inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,, +milvus.go.memstats.heap.objects,gauge,,,,Number of allocated objects.,0,milvus,, +milvus.go.memstats.heap.released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,, +milvus.go.memstats.heap.sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,, +milvus.go.memstats.last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,, +milvus.go.memstats.lookups_total,count,,,,Total number of pointer lookups.,0,milvus,, +milvus.go.memstats.mallocs_total,count,,,,Total number of mallocs.,0,milvus,, +milvus.go.memstats.mcache.inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,, +milvus.go.memstats.mcache.sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,, +milvus.go.memstats.mspan.inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,, +milvus.go.memstats.mspan.sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,, +milvus.go.memstats.next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,, +milvus.go.memstats.other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,, +milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,, +milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,, +milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,, +milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,, +milvus.graph_search_cnt,gauge,,,,number of graph search per request,0,milvus,, +milvus.hnsw.bitset_ratio,gauge,,,,HNSW bitset ratio for search and range search,0,milvus,, +milvus.hnsw.search_hops,gauge,,,,HNSW search hops in layer 0,0,milvus,, +milvus.internal.core_search_latency,gauge,,,,[cpp]latency(us) of search on segment,0,milvus,, +milvus.internal.mmap.allocated_space_bytes,gauge,,,,[cpp]mmap allocated space stats,0,milvus,, +milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,, +milvus.internal.storage.kv_size,gauge,,,,[cpp]kv size stats,0,milvus,, +milvus.internal.storage.load_duration,gauge,,,,[cpp]durations of load segment,0,milvus,, +milvus.internal.storage.op_count,count,,,,[cpp]count of persistent data operation,0,milvus,, +milvus.internal.storage.request_latency,gauge,,,,[cpp]request latency(ms) on the client side,0,milvus,, +milvus.io_cnt,gauge,,,,io cnt per request,0,milvus,, +milvus.ivf_search_cnt,gauge,,,,number of ivf search per request,0,milvus,, +milvus.load_latency,gauge,,,,index load latency (ms),0,milvus,, +milvus.process.cpu_seconds_total,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, +milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, +milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, +milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,, +milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,, +milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,, +milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,, +milvus.quant.compute_cnt,gauge,,,,quant compute cnt per request,0,milvus,, +milvus.queue.latency,gauge,,,,queue latency per request,0,milvus,, +milvus.range_search_latency,gauge,,,,range search latency (ms),0,milvus,, +milvus.raw_compute_cnt,gauge,,,,raw compute cnt per request,0,milvus,, +milvus.re_search_cnt,gauge,,,,number of fallback search per request,0,milvus,, +milvus.search.latency,gauge,,,,search latency (ms),0,milvus,, +milvus.search.topk,gauge,,,,search topk,0,milvus,, From 8c91841fec27f6679b91a1b986373edfde9ecc9c Mon Sep 17 00:00:00 2001 From: David Kirov Date: Wed, 27 Nov 2024 19:31:46 +0100 Subject: [PATCH 15/31] Update metrics map --- milvus/datadog_checks/milvus/metrics.py | 71 ++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py index 24b40a7fc7218..305e11cff621b 100644 --- a/milvus/datadog_checks/milvus/metrics.py +++ b/milvus/datadog_checks/milvus/metrics.py @@ -150,10 +150,77 @@ 'milvus_rootcoord_timestamp_saved': 'rootcoord.timestamp_saved', 'milvus_runtime_info': 'runtime_info', 'milvus_storage_kv_size': 'storage.kv_size', - 'milvus_storage_op_count': 'storage.op_count', + 'milvus_storage_op_count': 'storage.op', 'milvus_storage_request_latency': 'storage.request_latency', + 'milvus_bf_search_cnt': 'milvus.bf_search_cnt', + 'bitset_ratio': 'bitset_ratio', + 'build_latency': 'build_latency', + 'cache_hit_cnt': 'cache_hit_cnt', + 'diskann_range_search_iters': 'diskann.range_search_iters', + 'diskann_search_hops': 'diskann.search_hops', + 'diskann_bitset_ratio': 'diskann_bitset_ratio', + 'exec_latency': 'exec_latency', + 'filter_connectivity_ratio': 'filter.connectivity_ratio', + 'filter_mv_activated_fields_cnt': 'filter.mv.activated_fields_cnt', + 'filter_mv_change_base_cnt': 'filter.mv.change_base_cnt', + 'filter_mv_only_cnt': 'filter.mv.only_cnt', + 'filter_mv_supplement_ep_bool_cnt': 'filter.mv.supplement_ep_bool_cnt', + 'go_gc_duration_seconds': 'go.gc_duration_seconds', + 'go_goroutines': 'go.goroutines', + 'go_info': 'go.info', + 'go_memstats_alloc_bytes': 'go.memstats.alloc_bytes', + 'go_memstats_alloc_bytes_total': 'go.memstats.alloc_bytes_total', + 'go_memstats_buck_hash_sys_bytes': 'go.memstats.buck_hash_sys_bytes', + 'go_memstats_frees_total': 'go.memstats.frees_total', + 'go_memstats_gc_sys_bytes': 'go.memstats.gc_sys_bytes', + 'go_memstats_heap_alloc_bytes': 'go.memstats.heap.alloc_bytes', + 'go_memstats_heap_idle_bytes': 'go.memstats.heap.idle_bytes', + 'go_memstats_heap_inuse_bytes': 'go.memstats.heap.inuse_bytes', + 'go_memstats_heap_objects': 'go.memstats.heap.objects', + 'go_memstats_heap_released_bytes': 'go.memstats.heap.released_bytes', + 'go_memstats_heap_sys_bytes': 'go.memstats.heap.sys_bytes', + 'go_memstats_last_gc_time_seconds': 'go.memstats.last_gc_time_seconds', + 'go_memstats_lookups_total': 'go.memstats.lookups_total', + 'go_memstats_mallocs_total': 'go.memstats.mallocs_total', + 'go_memstats_mcache_inuse_bytes': 'go.memstats.mcache.inuse_bytes', + 'go_memstats_mcache_sys_bytes': 'go.memstats.mcache.sys_bytes', + 'go_memstats_mspan_inuse_bytes': 'go.memstats.mspan.inuse_bytes', + 'go_memstats_mspan_sys_bytes': 'go.memstats.mspan.sys_bytes', + 'go_memstats_next_gc_bytes': 'go.memstats.next_gc_bytes', + 'go_memstats_other_sys_bytes': 'go.memstats.other_sys_bytes', + 'go_memstats_stack_inuse_bytes': 'go.memstats.stack.inuse_bytes', + 'go_memstats_stack_sys_bytes': 'go.memstats.stack.sys_bytes', + 'go_memstats_sys_bytes': 'go.memstats.sys_bytes', + 'go_threads': 'go.threads', + 'graph_search_cnt': 'graph_search_cnt', + 'hnsw_bitset_ratio': 'hnsw.bitset_ratio', + 'hnsw_search_hops': 'hnsw.search_hops', + 'internal_core_search_latency': 'internal.core_search_latency', + 'internal_mmap_allocated_space_bytes': 'internal.mmap.allocated_space_bytes', + 'internal_mmap_in_used_space_bytes': 'internal.mmap.in_used_space_bytes', + 'internal_storage_kv_size': 'internal.storage.kv_size', + 'internal_storage_load_duration': 'internal.storage.load_duration', + 'internal_storage_op_count': 'internal.storage.op_count', + 'internal_storage_request_latency': 'internal.storage.request_latency', + 'io_cnt': 'io_cnt', + 'ivf_search_cnt': 'ivf_search_cnt', + 'load_latency': 'load_latency', + 'process_cpu_seconds_total': 'process.cpu_seconds_total', + 'process_max_fds': 'process.max_fds', + 'process_open_fds': 'process.open_fds', + 'process_resident_memory_bytes': 'process.resident_memory_bytes', + 'process_start_time_seconds': 'process.start_time_seconds', + 'process_virtual_memory_bytes': 'process.virtual_memory.bytes', + 'process_virtual_memory_max_bytes': 'process.virtual_memory.max_bytes', + 'quant_compute_cnt': 'quant.compute_cnt', + 'queue_latency': 'queue.latency', + 'range_search_latency': 'range_search_latency', + 'raw_compute_cnt': 'raw_compute_cnt', + 're_search_cnt': 're_search_cnt', + 'search_latency': 'search.latency', + 'search_topk': 'search.topk', } RENAME_LABELS_MAP = { - "version": "milvus_version", + 'version': 'milvus_version', } From 65b8c37e6ed8856cb1937deb2f981faa69231bcb Mon Sep 17 00:00:00 2001 From: David Kirov Date: Wed, 27 Nov 2024 19:59:41 +0100 Subject: [PATCH 16/31] Expand histogram metrics --- milvus/datadog_checks/milvus/metrics.py | 338 ++++++++++++++++++------ milvus/metadata.csv | 336 +++++++++++++++++------ 2 files changed, 505 insertions(+), 169 deletions(-) diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py index 305e11cff621b..8ba8a77036d3c 100644 --- a/milvus/datadog_checks/milvus/metrics.py +++ b/milvus/datadog_checks/milvus/metrics.py @@ -5,8 +5,12 @@ METRIC_MAP = { 'milvus_build_info': 'build_info', 'milvus_cgo_active_future_total': 'cgo.active_future_total', - 'milvus_cgo_cgo_duration_seconds': 'cgo.cgo_duration_seconds', - 'milvus_cgo_cgo_queue_duration_seconds': 'cgo.cgo_queue_duration_seconds', + 'milvus_cgo_cgo_duration_seconds_sum': 'cgo.cgo_duration_seconds.sum', + 'milvus_cgo_cgo_duration_seconds_count': 'cgo.cgo_duration_seconds.count', + 'milvus_cgo_cgo_duration_seconds_bucket': 'cgo.cgo_duration_seconds.bucket', + 'milvus_cgo_cgo_queue_duration_seconds_sum': 'cgo.cgo_queue_duration_seconds.sum', + 'milvus_cgo_cgo_queue_duration_seconds_count': 'cgo.cgo_queue_duration_seconds.count', + 'milvus_cgo_cgo_queue_duration_seconds_bucket': 'cgo.cgo_queue_duration_seconds.bucket', 'milvus_cgo_running_cgo_call_total': 'cgo.running_cgo_call_total', 'milvus_datacoord_channel_checkpoint_unix_seconds': 'datacoord.channel_checkpoint_unix_seconds', 'milvus_datacoord_collection_num': 'datacoord.collection_num', @@ -20,13 +24,17 @@ 'milvus_datacoord_stored_binlog_size': 'datacoord.stored.binlog_size', 'milvus_datacoord_stored_index_files_size': 'datacoord.stored.index_files_size', 'milvus_datacoord_stored_rows_num': 'datacoord.stored.rows_num', - 'milvus_datacoord_task_execute_max_latency': 'datacoord.task_execute_max_latency', + 'milvus_datacoord_task_execute_max_latency_sum': 'datacoord.task_execute_max_latency.sum', + 'milvus_datacoord_task_execute_max_latency_count': 'datacoord.task_execute_max_latency.count', + 'milvus_datacoord_task_execute_max_latency_bucket': 'datacoord.task_execute_max_latency.bucket', 'milvus_datacoord_watched_dml_chanel_num': 'datacoord.watched_dml_chanel_num', 'milvus_datanode_autoflush_buffer_op_count': 'datanode.autoflush_buffer_op_count', 'milvus_datanode_consume_bytes_count': 'datanode.consume.bytes_count', 'milvus_datanode_consume_msg_count': 'datanode.consume.msg_count', 'milvus_datanode_consume_tt_lag_ms': 'datanode.consume.tt_lag_ms', - 'milvus_datanode_encode_buffer_latency': 'datanode.encode_buffer_latency', + 'milvus_datanode_encode_buffer_latency_sum': 'datanode.encode_buffer_latency.sum', + 'milvus_datanode_encode_buffer_latency_count': 'datanode.encode_buffer_latency.count', + 'milvus_datanode_encode_buffer_latency_bucket': 'datanode.encode_buffer_latency.bucket', 'milvus_datanode_flowgraph_num': 'datanode.flowgraph_num', 'milvus_datanode_flush_buffer_op_count': 'datanode.flush.buffer_op_count', 'milvus_datanode_flush_req_count': 'datanode.flush.req_count', @@ -34,61 +42,113 @@ 'milvus_datanode_flushed_data_size': 'datanode.flushed_data.size', 'milvus_datanode_msg_dispatcher_tt_lag_ms': 'datanode.msg.dispatcher_tt_lag_ms', 'milvus_datanode_msg_rows_count': 'datanode.msg.rows_count', - 'milvus_datanode_save_latency': 'datanode.save_latency', - 'milvus_flushed_segment_file_num': 'flushed_segment_file_num', - 'milvus_indexnode_build_index_latency': 'indexnode.build_index_latency', - 'milvus_indexnode_encode_index_latency': 'indexnode.encode_index_latency', + 'milvus_datanode_save_latency_sum': 'datanode.save_latency.sum', + 'milvus_datanode_save_latency_count': 'datanode.save_latency.count', + 'milvus_datanode_save_latency_bucket': 'datanode.save_latency.bucket', + 'milvus_flushed_segment_file_num_sum': 'flushed_segment_file_num.sum', + 'milvus_flushed_segment_file_num_count': 'flushed_segment_file_num.count', + 'milvus_flushed_segment_file_num_bucket': 'flushed_segment_file_num.bucket', + 'milvus_indexnode_build_index_latency_sum': 'indexnode.build_index_latency.sum', + 'milvus_indexnode_build_index_latency_count': 'indexnode.build_index_latency.count', + 'milvus_indexnode_build_index_latency_bucket': 'indexnode.build_index_latency.bucket', + 'milvus_indexnode_encode_index_latency_sum': 'indexnode.encode_index_latency.sum', + 'milvus_indexnode_encode_index_latency_count': 'indexnode.encode_index_latency.count', + 'milvus_indexnode_encode_index_latency_bucket': 'indexnode.encode_index_latency.bucket', 'milvus_indexnode_index_task_count': 'indexnode.index.task_count', - 'milvus_indexnode_index_task_latency_in_queue': 'indexnode.index.task_latency_in_queue', - 'milvus_indexnode_knowhere_build_index_latency': 'indexnode.knowhere_build_index_latency', - 'milvus_indexnode_save_index_latency': 'indexnode.save_index_latency', - 'milvus_meta_kv_size': 'meta.kv_size', + 'milvus_indexnode_index_task_latency_in_queue_sum': 'indexnode.index.task_latency_in_queue.sum', + 'milvus_indexnode_index_task_latency_in_queue_count': 'indexnode.index.task_latency_in_queue.count', + 'milvus_indexnode_index_task_latency_in_queue_bucket': 'indexnode.index.task_latency_in_queue.bucket', + 'milvus_indexnode_knowhere_build_index_latency_sum': 'indexnode.knowhere_build_index_latency.sum', + 'milvus_indexnode_knowhere_build_index_latency_count': 'indexnode.knowhere_build_index_latency.count', + 'milvus_indexnode_knowhere_build_index_latency_bucket': 'indexnode.knowhere_build_index_latency.bucket', + 'milvus_indexnode_save_index_latency_sum': 'indexnode.save_index_latency.sum', + 'milvus_indexnode_save_index_latency_count': 'indexnode.save_index_latency.count', + 'milvus_indexnode_save_index_latency_bucket': 'indexnode.save_index_latency.bucket', + 'milvus_meta_kv_size_sum': 'meta.kv_size.sum', + 'milvus_meta_kv_size_count': 'meta.kv_size.count', + 'milvus_meta_kv_size_bucket': 'meta.kv_size.bucket', 'milvus_meta_op_count': 'meta.op_count', - 'milvus_meta_request_latency': 'meta.request_latency', + 'milvus_meta_request_latency_sum': 'meta.request_latency.sum', + 'milvus_meta_request_latency_count': 'meta.request_latency.count', + 'milvus_meta_request_latency_bucket': 'meta.request_latency.bucket', 'milvus_msg_queue_consumer_num': 'msg_queue_consumer_num', 'milvus_msgstream_op_count': 'msgstream.op_count', - 'milvus_msgstream_request_latency': 'msgstream.request_latency', + 'milvus_msgstream_request_latency_sum': 'msgstream.request_latency.sum', + 'milvus_msgstream_request_latency_count': 'msgstream.request_latency.count', + 'milvus_msgstream_request_latency_bucket': 'msgstream.request_latency.bucket', 'milvus_num_node': 'num_node', - 'milvus_proxy_apply_pk_latency': 'proxy.apply.pk_latency', - 'milvus_proxy_apply_timestamp_latency': 'proxy.apply.timestamp_latency', - 'milvus_proxy_assign_segmentID_latency': 'proxy.assign_segmentID_latency', + 'milvus_proxy_apply_pk_latency_sum': 'proxy.apply.pk_latency.sum', + 'milvus_proxy_apply_pk_latency_count': 'proxy.apply.pk_latency.count', + 'milvus_proxy_apply_pk_latency_bucket': 'proxy.apply.pk_latency.bucket', + 'milvus_proxy_apply_timestamp_latency_sum': 'proxy.apply.timestamp_latency.sum', + 'milvus_proxy_apply_timestamp_latency_count': 'proxy.apply.timestamp_latency.count', + 'milvus_proxy_apply_timestamp_latency_bucket': 'proxy.apply.timestamp_latency.bucket', + 'milvus_proxy_assign_segmentID_latency_sum': 'proxy.assign_segmentID_latency.sum', + 'milvus_proxy_assign_segmentID_latency_count': 'proxy.assign_segmentID_latency.count', + 'milvus_proxy_assign_segmentID_latency_bucket': 'proxy.assign_segmentID_latency.bucket', 'milvus_proxy_cache_hit_count': 'proxy.cache.hit_count', - 'milvus_proxy_cache_update_latency': 'proxy.cache.update_latency', + 'milvus_proxy_cache_update_latency_sum': 'proxy.cache.update_latency.sum', + 'milvus_proxy_cache_update_latency_count': 'proxy.cache.update_latency.count', + 'milvus_proxy_cache_update_latency_bucket': 'proxy.cache.update_latency.bucket', 'milvus_proxy_delete_vectors_count': 'proxy.delete_vectors_count', 'milvus_proxy_msgstream_obj_num': 'proxy.msgstream_obj_num', - 'milvus_proxy_mutation_send_latency': 'proxy.mutation_send_latency', + 'milvus_proxy_mutation_send_latency_sum': 'proxy.mutation_send_latency.sum', + 'milvus_proxy_mutation_send_latency_count': 'proxy.mutation_send_latency.count', + 'milvus_proxy_mutation_send_latency_bucket': 'proxy.mutation_send_latency.bucket', 'milvus_proxy_rate_limit_req_count': 'proxy.rate_limit_req_count', 'milvus_proxy_report_value': 'proxy.report_value', 'milvus_proxy_req_count': 'proxy.req.count', - 'milvus_proxy_req_in_queue_latency': 'proxy.req.in_queue_latency', - 'milvus_proxy_req_latency': 'proxy.req.latency', + 'milvus_proxy_req_in_queue_latency_sum': 'proxy.req.in_queue_latency.sum', + 'milvus_proxy_req_in_queue_latency_count': 'proxy.req.in_queue_latency.count', + 'milvus_proxy_req_in_queue_latency_bucket': 'proxy.req.in_queue_latency.bucket', + 'milvus_proxy_req_latency_sum': 'proxy.req.latency.sum', + 'milvus_proxy_req_latency_count': 'proxy.req.latency.count', + 'milvus_proxy_req_latency_bucket': 'proxy.req.latency.bucket', 'milvus_proxy_send_bytes_count': 'proxy.send_bytes_count', - 'milvus_proxy_sq_decode_result_latency': 'proxy.sq.decode_result_latency', - 'milvus_proxy_sq_reduce_result_latency': 'proxy.sq.reduce_result_latency', - 'milvus_proxy_sq_wait_result_latency': 'proxy.sq.wait_result_latency', - 'milvus_proxy_sync_segment_request_length': 'proxy.sync_segment_request_length', + 'milvus_proxy_sq_decode_result_latency_sum': 'proxy.sq.decode_result_latency.sum', + 'milvus_proxy_sq_decode_result_latency_count': 'proxy.sq.decode_result_latency.count', + 'milvus_proxy_sq_decode_result_latency_bucket': 'proxy.sq.decode_result_latency.bucket', + 'milvus_proxy_sq_reduce_result_latency_sum': 'proxy.sq.reduce_result_latency.sum', + 'milvus_proxy_sq_reduce_result_latency_count': 'proxy.sq.reduce_result_latency.count', + 'milvus_proxy_sq_reduce_result_latency_bucket': 'proxy.sq.reduce_result_latency.bucket', + 'milvus_proxy_sq_wait_result_latency_sum': 'proxy.sq.wait_result_latency.sum', + 'milvus_proxy_sq_wait_result_latency_count': 'proxy.sq.wait_result_latency.count', + 'milvus_proxy_sq_wait_result_latency_bucket': 'proxy.sq.wait_result_latency.bucket', + 'milvus_proxy_sync_segment_request_length_sum': 'proxy.sync_segment_request_length.sum', + 'milvus_proxy_sync_segment_request_length_count': 'proxy.sync_segment_request_length.count', + 'milvus_proxy_sync_segment_request_length_bucket': 'proxy.sync_segment_request_length.bucket', 'milvus_proxy_tt_lag_ms': 'proxy.tt_lag_ms', 'milvus_querycoord_collection_num': 'querycoord.collection_num', 'milvus_querycoord_current_target_checkpoint_unix_seconds': 'querycoord.current_target_checkpoint_unix_seconds', - 'milvus_querycoord_load_latency': 'querycoord.load.latency', + 'milvus_querycoord_load_latency_sum': 'querycoord.load.latency.sum', + 'milvus_querycoord_load_latency_count': 'querycoord.load.latency.count', + 'milvus_querycoord_load_latency_bucket': 'querycoord.load.latency.bucket', 'milvus_querycoord_load_req_count': 'querycoord.load.req_count', 'milvus_querycoord_partition_num': 'querycoord.partition_num', 'milvus_querycoord_querynode_num': 'querycoord.querynode_num', - 'milvus_querycoord_release_latency': 'querycoord.release.latency', + 'milvus_querycoord_release_latency_sum': 'querycoord.release.latency.sum', + 'milvus_querycoord_release_latency_count': 'querycoord.release.latency.count', + 'milvus_querycoord_release_latency_bucket': 'querycoord.release.latency.bucket', 'milvus_querycoord_release_req_count': 'querycoord.release.req_count', 'milvus_querycoord_task_num': 'querycoord_task_num', - 'milvus_querynode_apply_bf_latency': 'querynode.apply_bf_latency', + 'milvus_querynode_apply_bf_latency_sum': 'querynode.apply_bf_latency.sum', + 'milvus_querynode_apply_bf_latency_count': 'querynode.apply_bf_latency.count', + 'milvus_querynode_apply_bf_latency_bucket': 'querynode.apply_bf_latency.bucket', 'milvus_querynode_collection_num': 'querynode.collection_num', 'milvus_querynode_consume_bytes_counter': 'querynode.consume.bytes_counter', 'milvus_querynode_consume_msg_count': 'querynode.consume.msg_count', 'milvus_querynode_consume_tt_lag_ms': 'querynode.consume.tt_lag_ms', 'milvus_querynode_disk_cache_evict_bytes': 'querynode.disk.cache.evict.bytes', 'milvus_querynode_disk_cache_evict_duration': 'querynode.disk.cache.evict.duration', - 'milvus_querynode_disk_cache_evict_global_duration': 'querynode.disk.cache.evict.global_duration', + 'milvus_querynode_disk_cache_evict_global_duration_sum': 'querynode.disk.cache.evict.global_duration.sum', + 'milvus_querynode_disk_cache_evict_global_duration_count': 'querynode.disk.cache.evict.global_duration.count', + 'milvus_querynode_disk_cache_evict_global_duration_bucket': 'querynode.disk.cache.evict.global_duration.bucket', 'milvus_querynode_disk_cache_evict_total': 'querynode.disk.cache.evict.total', 'milvus_querynode_disk_cache_load_bytes': 'querynode.disk.cache.load.bytes', 'milvus_querynode_disk_cache_load_duration': 'querynode.disk.cache.load.duration', - 'milvus_querynode_disk_cache_load_global_duration': 'querynode.disk.cache.load.global_duration', + 'milvus_querynode_disk_cache_load_global_duration_sum': 'querynode.disk.cache.load.global_duration.sum', + 'milvus_querynode_disk_cache_load_global_duration_count': 'querynode.disk.cache.load.global_duration.count', + 'milvus_querynode_disk_cache_load_global_duration_bucket': 'querynode.disk.cache.load.global_duration.bucket', 'milvus_querynode_disk_cache_load_total': 'querynode.disk.cache.load.total', 'milvus_querynode_disk_used_size': 'querynode.disk.used_size', 'milvus_querynode_dml_vchannel_num': 'querynode.dml_vchannel_num', @@ -96,44 +156,88 @@ 'milvus_querynode_entity_size': 'querynode.entity.size', 'milvus_querynode_execute_bytes_counter': 'querynode.execute_bytes_counter', 'milvus_querynode_flowgraph_num': 'querynode.flowgraph_num', - 'milvus_querynode_forward_delete_latency': 'querynode.forward_delete_latency', - 'milvus_querynode_load_index_latency': 'querynode.load.index_latency', + 'milvus_querynode_forward_delete_latency_sum': 'querynode.forward_delete_latency.sum', + 'milvus_querynode_forward_delete_latency_count': 'querynode.forward_delete_latency.count', + 'milvus_querynode_forward_delete_latency_bucket': 'querynode.forward_delete_latency.bucket', + 'milvus_querynode_load_index_latency_sum': 'querynode.load.index_latency.sum', + 'milvus_querynode_load_index_latency_count': 'querynode.load.index_latency.count', + 'milvus_querynode_load_index_latency_bucket': 'querynode.load.index_latency.bucket', 'milvus_querynode_load_segment_concurrency': 'querynode.load.segment.concurrency', - 'milvus_querynode_load_segment_latency': 'querynode.load.segment.latency', + 'milvus_querynode_load_segment_latency_sum': 'querynode.load.segment.latency.sum', + 'milvus_querynode_load_segment_latency_count': 'querynode.load.segment.latency.count', + 'milvus_querynode_load_segment_latency_bucket': 'querynode.load.segment.latency.bucket', 'milvus_querynode_msg_dispatcher_tt_lag_ms': 'querynode.msg_dispatcher_tt_lag_ms', 'milvus_querynode_partition_num': 'querynode.partition_num', - 'milvus_querynode_process_insert_or_delete_latency': 'querynode.process_insert_or_delete_latency', + 'milvus_querynode_process_insert_or_delete_latency_sum': 'querynode.process_insert_or_delete_latency.sum', + 'milvus_querynode_process_insert_or_delete_latency_count': 'querynode.process_insert_or_delete_latency.count', + 'milvus_querynode_process_insert_or_delete_latency_bucket': 'querynode.process_insert_or_delete_latency.bucket', 'milvus_querynode_read_task_concurrency': 'querynode.read_task.concurrency', 'milvus_querynode_read_task_ready_len': 'querynode.read_task.ready_len', 'milvus_querynode_read_task_unsolved_len': 'querynode.read_task.unsolved_len', - 'milvus_querynode_search_group_nq': 'querynode.search.group.nq', - 'milvus_querynode_search_group_size': 'querynode.search.group.size', - 'milvus_querynode_search_group_topk': 'querynode.search.group.topk', - 'milvus_querynode_search_nq': 'querynode.search.nq', - 'milvus_querynode_search_topk': 'querynode.search.topk', + 'milvus_querynode_search_group_nq_sum': 'querynode.search.group.nq.sum', + 'milvus_querynode_search_group_nq_count': 'querynode.search.group.nq.count', + 'milvus_querynode_search_group_nq_bucket': 'querynode.search.group.nq.bucket', + 'milvus_querynode_search_group_size_sum': 'querynode.search.group.size.sum', + 'milvus_querynode_search_group_size_count': 'querynode.search.group.size.count', + 'milvus_querynode_search_group_size_bucket': 'querynode.search.group.size.bucket', + 'milvus_querynode_search_group_topk_sum': 'querynode.search.group.topk.sum', + 'milvus_querynode_search_group_topk_count': 'querynode.search.group.topk.count', + 'milvus_querynode_search_group_topk_bucket': 'querynode.search.group.topk.bucket', + 'milvus_querynode_search_nq_sum': 'querynode.search.nq.sum', + 'milvus_querynode_search_nq_count': 'querynode.search.nq.count', + 'milvus_querynode_search_nq_bucket': 'querynode.search.nq.bucket', + 'milvus_querynode_search_topk_sum': 'querynode.search.topk.sum', + 'milvus_querynode_search_topk_count': 'querynode.search.topk.count', + 'milvus_querynode_search_topk_bucket': 'querynode.search.topk.bucket', 'milvus_querynode_segment_access_duration': 'querynode.segment.access.duration', - 'milvus_querynode_segment_access_global_duration': 'querynode.segment.access.global_duration', + 'milvus_querynode_segment_access_global_duration_sum': 'querynode.segment.access.global_duration.sum', + 'milvus_querynode_segment_access_global_duration_count': 'querynode.segment.access.global_duration.count', + 'milvus_querynode_segment_access_global_duration_bucket': 'querynode.segment.access.global_duration.bucket', 'milvus_querynode_segment_access_total': 'querynode.segment.access.total', 'milvus_querynode_segment_access_wait_cache_duration': 'querynode.segment.access.wait_cache.duration', - 'milvus_querynode_segment_access_wait_cache_global_duration': 'querynode.segment.access.wait_cache.global_duration', + 'milvus_querynode_segment_access_wait_cache_global_duration_sum': 'querynode.segment.access.wait_cache.global_duration.sum', + 'milvus_querynode_segment_access_wait_cache_global_duration_count': 'querynode.segment.access.wait_cache.global_duration.count', + 'milvus_querynode_segment_access_wait_cache_global_duration_bucket': 'querynode.segment.access.wait_cache.global_duration.bucket', 'milvus_querynode_segment_access_wait_cache_total': 'querynode.segment.access.wait_cache.total', - 'milvus_querynode_segment_latency_per_vector': 'querynode.segment.latency_per_vector', + 'milvus_querynode_segment_latency_per_vector_sum': 'querynode.segment.latency_per_vector.sum', + 'milvus_querynode_segment_latency_per_vector_count': 'querynode.segment.latency_per_vector.count', + 'milvus_querynode_segment_latency_per_vector_bucket': 'querynode.segment.latency_per_vector.bucket', 'milvus_querynode_segment_num': 'querynode.segment.num', - 'milvus_querynode_sq_core_latency': 'querynode.sq.core_latency', - 'milvus_querynode_sq_queue_latency': 'querynode.sq.queue.latency', - 'milvus_querynode_sq_queue_user_latency': 'querynode.sq.queue.user_latency', - 'milvus_querynode_sq_reduce_latency': 'querynode.sq.reduce_latency', + 'milvus_querynode_sq_core_latency_sum': 'querynode.sq.core_latency.sum', + 'milvus_querynode_sq_core_latency_count': 'querynode.sq.core_latency.count', + 'milvus_querynode_sq_core_latency_bucket': 'querynode.sq.core_latency.bucket', + 'milvus_querynode_sq_queue_latency_sum': 'querynode.sq.queue.latency.sum', + 'milvus_querynode_sq_queue_latency_count': 'querynode.sq.queue.latency.count', + 'milvus_querynode_sq_queue_latency_bucket': 'querynode.sq.queue.latency.bucket', + 'milvus_querynode_sq_queue_user_latency_sum': 'querynode.sq.queue.user_latency.sum', + 'milvus_querynode_sq_queue_user_latency_count': 'querynode.sq.queue.user_latency.count', + 'milvus_querynode_sq_queue_user_latency_bucket': 'querynode.sq.queue.user_latency.bucket', + 'milvus_querynode_sq_reduce_latency_sum': 'querynode.sq.reduce_latency.sum', + 'milvus_querynode_sq_reduce_latency_count': 'querynode.sq.reduce_latency.count', + 'milvus_querynode_sq_reduce_latency_bucket': 'querynode.sq.reduce_latency.bucket', 'milvus_querynode_sq_req_count': 'querynode.sq.req.count', - 'milvus_querynode_sq_req_latency': 'querynode.sq.req.latency', - 'milvus_querynode_sq_segment_latency': 'querynode.sq.segment_latency', - 'milvus_querynode_sq_wait_tsafe_latency': 'querynode.sq.wait_tsafe_latency', + 'milvus_querynode_sq_req_latency_sum': 'querynode.sq.req.latency.sum', + 'milvus_querynode_sq_req_latency_count': 'querynode.sq.req.latency.count', + 'milvus_querynode_sq_req_latency_bucket': 'querynode.sq.req.latency.bucket', + 'milvus_querynode_sq_segment_latency_sum': 'querynode.sq.segment_latency.sum', + 'milvus_querynode_sq_segment_latency_count': 'querynode.sq.segment_latency.count', + 'milvus_querynode_sq_segment_latency_bucket': 'querynode.sq.segment_latency.bucket', + 'milvus_querynode_sq_wait_tsafe_latency_sum': 'querynode.sq.wait_tsafe_latency.sum', + 'milvus_querynode_sq_wait_tsafe_latency_count': 'querynode.sq.wait_tsafe_latency.count', + 'milvus_querynode_sq_wait_tsafe_latency_bucket': 'querynode.sq.wait_tsafe_latency.bucket', 'milvus_querynode_wait_processing_msg_count': 'querynode.wait_processing_msg_count', - 'milvus_querynode_watch_dml_channel_latency': 'querynode.watch_dml_channel_latency', + 'milvus_querynode_watch_dml_channel_latency_sum': 'querynode.watch_dml_channel_latency.sum', + 'milvus_querynode_watch_dml_channel_latency_count': 'querynode.watch_dml_channel_latency.count', + 'milvus_querynode_watch_dml_channel_latency_bucket': 'querynode.watch_dml_channel_latency.bucket', 'milvus_rootcoord_collection_num': 'rootcoord.collection_num', 'milvus_rootcoord_credential_num': 'rootcoord.credential_num', 'milvus_rootcoord_ddl_req_count': 'rootcoord.ddl_req.count', - 'milvus_rootcoord_ddl_req_latency': 'rootcoord.ddl_req.latency', - 'milvus_rootcoord_ddl_req_latency_in_queue': 'rootcoord.ddl_req.latency_in_queue', + 'milvus_rootcoord_ddl_req_latency_sum': 'rootcoord.ddl_req.latency.sum', + 'milvus_rootcoord_ddl_req_latency_count': 'rootcoord.ddl_req.latency.count', + 'milvus_rootcoord_ddl_req_latency_bucket': 'rootcoord.ddl_req.latency.bucket', + 'milvus_rootcoord_ddl_req_latency_in_queue_sum': 'rootcoord.ddl_req.latency_in_queue.sum', + 'milvus_rootcoord_ddl_req_latency_in_queue_count': 'rootcoord.ddl_req.latency_in_queue.count', + 'milvus_rootcoord_ddl_req_latency_in_queue_bucket': 'rootcoord.ddl_req.latency_in_queue.bucket', 'milvus_rootcoord_dml_channel_num': 'rootcoord.dml_channel_num', 'milvus_rootcoord_entity_num': 'rootcoord.entity_num', 'milvus_rootcoord_force_deny_writing_counter': 'rootcoord.force_deny_writing_counter', @@ -145,26 +249,54 @@ 'milvus_rootcoord_produce_tt_lag_ms': 'rootcoord.produce_tt_lag_ms', 'milvus_rootcoord_proxy_num': 'rootcoord.proxy_num', 'milvus_rootcoord_qn_mem_high_water_level': 'rootcoord.qn_mem_high_water_level', - 'milvus_rootcoord_sync_timetick_latency': 'rootcoord.sync_timetick_latency', + 'milvus_rootcoord_sync_timetick_latency_sum': 'rootcoord.sync_timetick_latency.sum', + 'milvus_rootcoord_sync_timetick_latency_count': 'rootcoord.sync_timetick_latency.count', + 'milvus_rootcoord_sync_timetick_latency_bucket': 'rootcoord.sync_timetick_latency.bucket', 'milvus_rootcoord_timestamp': 'rootcoord.timestamp', 'milvus_rootcoord_timestamp_saved': 'rootcoord.timestamp_saved', 'milvus_runtime_info': 'runtime_info', 'milvus_storage_kv_size': 'storage.kv_size', - 'milvus_storage_op_count': 'storage.op', + 'milvus_storage_op_count': 'storage.op_count', 'milvus_storage_request_latency': 'storage.request_latency', - 'milvus_bf_search_cnt': 'milvus.bf_search_cnt', - 'bitset_ratio': 'bitset_ratio', - 'build_latency': 'build_latency', - 'cache_hit_cnt': 'cache_hit_cnt', - 'diskann_range_search_iters': 'diskann.range_search_iters', - 'diskann_search_hops': 'diskann.search_hops', - 'diskann_bitset_ratio': 'diskann_bitset_ratio', - 'exec_latency': 'exec_latency', - 'filter_connectivity_ratio': 'filter.connectivity_ratio', - 'filter_mv_activated_fields_cnt': 'filter.mv.activated_fields_cnt', - 'filter_mv_change_base_cnt': 'filter.mv.change_base_cnt', - 'filter_mv_only_cnt': 'filter.mv.only_cnt', - 'filter_mv_supplement_ep_bool_cnt': 'filter.mv.supplement_ep_bool_cnt', + 'bf_search_cnt_sum': 'bf_search_cnt.sum', + 'bf_search_cnt_count': 'bf_search_cnt.count', + 'bf_search_cnt_bucket': 'bf_search_cnt.bucket', + 'bitset_ratio_sum': 'bitset_ratio.sum', + 'bitset_ratio_count': 'bitset_ratio.count', + 'bitset_ratio_bucket': 'bitset_ratio.bucket', + 'build_latency_sum': 'build_latency.sum', + 'build_latency_count': 'build_latency.count', + 'build_latency_bucket': 'build_latency.bucket', + 'cache_hit_cnt_sum': 'cache_hit_cnt.sum', + 'cache_hit_cnt_count': 'cache_hit_cnt.count', + 'cache_hit_cnt_bucket': 'cache_hit_cnt.bucket', + 'diskann_range_search_iters_sum': 'diskann.range_search_iters.sum', + 'diskann_range_search_iters_count': 'diskann.range_search_iters.count', + 'diskann_range_search_iters_bucket': 'diskann.range_search_iters.bucket', + 'diskann_search_hops_sum': 'diskann.search_hops.sum', + 'diskann_search_hops_count': 'diskann.search_hops.count', + 'diskann_search_hops_bucket': 'diskann.search_hops.bucket', + 'diskann_bitset_ratio_sum': 'diskann_bitset_ratio.sum', + 'diskann_bitset_ratio_count': 'diskann_bitset_ratio.count', + 'diskann_bitset_ratio_bucket': 'diskann_bitset_ratio.bucket', + 'exec_latency_sum': 'exec_latency.sum', + 'exec_latency_count': 'exec_latency.count', + 'exec_latency_bucket': 'exec_latency.bucket', + 'filter_connectivity_ratio_sum': 'filter.connectivity_ratio.sum', + 'filter_connectivity_ratio_count': 'filter.connectivity_ratio.count', + 'filter_connectivity_ratio_bucket': 'filter.connectivity_ratio.bucket', + 'filter_mv_activated_fields_cnt_sum': 'filter.mv.activated_fields_cnt.sum', + 'filter_mv_activated_fields_cnt_count': 'filter.mv.activated_fields_cnt.count', + 'filter_mv_activated_fields_cnt_bucket': 'filter.mv.activated_fields_cnt.bucket', + 'filter_mv_change_base_cnt_sum': 'filter.mv.change_base_cnt.sum', + 'filter_mv_change_base_cnt_count': 'filter.mv.change_base_cnt.count', + 'filter_mv_change_base_cnt_bucket': 'filter.mv.change_base_cnt.bucket', + 'filter_mv_only_cnt_sum': 'filter.mv.only_cnt.sum', + 'filter_mv_only_cnt_count': 'filter.mv.only_cnt.count', + 'filter_mv_only_cnt_bucket': 'filter.mv.only_cnt.bucket', + 'filter_mv_supplement_ep_bool_cnt_sum': 'filter.mv.supplement_ep_bool_cnt.sum', + 'filter_mv_supplement_ep_bool_cnt_count': 'filter.mv.supplement_ep_bool_cnt.count', + 'filter_mv_supplement_ep_bool_cnt_bucket': 'filter.mv.supplement_ep_bool_cnt.bucket', 'go_gc_duration_seconds': 'go.gc_duration_seconds', 'go_goroutines': 'go.goroutines', 'go_info': 'go.info', @@ -192,19 +324,41 @@ 'go_memstats_stack_sys_bytes': 'go.memstats.stack.sys_bytes', 'go_memstats_sys_bytes': 'go.memstats.sys_bytes', 'go_threads': 'go.threads', - 'graph_search_cnt': 'graph_search_cnt', - 'hnsw_bitset_ratio': 'hnsw.bitset_ratio', - 'hnsw_search_hops': 'hnsw.search_hops', - 'internal_core_search_latency': 'internal.core_search_latency', - 'internal_mmap_allocated_space_bytes': 'internal.mmap.allocated_space_bytes', + 'graph_search_cnt_sum': 'graph_search_cnt.sum', + 'graph_search_cnt_count': 'graph_search_cnt.count', + 'graph_search_cnt_bucket': 'graph_search_cnt.bucket', + 'hnsw_bitset_ratio_sum': 'hnsw.bitset_ratio.sum', + 'hnsw_bitset_ratio_count': 'hnsw.bitset_ratio.count', + 'hnsw_bitset_ratio_bucket': 'hnsw.bitset_ratio.bucket', + 'hnsw_search_hops_sum': 'hnsw.search_hops.sum', + 'hnsw_search_hops_count': 'hnsw.search_hops.count', + 'hnsw_search_hops_bucket': 'hnsw.search_hops.bucket', + 'internal_core_search_latency_sum': 'internal.core_search_latency.sum', + 'internal_core_search_latency_count': 'internal.core_search_latency.count', + 'internal_core_search_latency_bucket': 'internal.core_search_latency.bucket', + 'internal_mmap_allocated_space_bytes_sum': 'internal.mmap.allocated_space_bytes.sum', + 'internal_mmap_allocated_space_bytes_count': 'internal.mmap.allocated_space_bytes.count', + 'internal_mmap_allocated_space_bytes_bucket': 'internal.mmap.allocated_space_bytes.bucket', 'internal_mmap_in_used_space_bytes': 'internal.mmap.in_used_space_bytes', - 'internal_storage_kv_size': 'internal.storage.kv_size', - 'internal_storage_load_duration': 'internal.storage.load_duration', + 'internal_storage_kv_size_sum': 'internal.storage.kv_size.sum', + 'internal_storage_kv_size_count': 'internal.storage.kv_size.count', + 'internal_storage_kv_size_bucket': 'internal.storage.kv_size.bucket', + 'internal_storage_load_duration_sum': 'internal.storage.load_duration.sum', + 'internal_storage_load_duration_count': 'internal.storage.load_duration.count', + 'internal_storage_load_duration_bucket': 'internal.storage.load_duration.bucket', 'internal_storage_op_count': 'internal.storage.op_count', - 'internal_storage_request_latency': 'internal.storage.request_latency', - 'io_cnt': 'io_cnt', - 'ivf_search_cnt': 'ivf_search_cnt', - 'load_latency': 'load_latency', + 'internal_storage_request_latency_sum': 'internal.storage.request_latency.sum', + 'internal_storage_request_latency_count': 'internal.storage.request_latency.count', + 'internal_storage_request_latency_bucket': 'internal.storage.request_latency.bucket', + 'io_cnt_sum': 'io_cnt.sum', + 'io_cnt_count': 'io_cnt.count', + 'io_cnt_bucket': 'io_cnt.bucket', + 'ivf_search_cnt_sum': 'ivf_search_cnt.sum', + 'ivf_search_cnt_count': 'ivf_search_cnt.count', + 'ivf_search_cnt_bucket': 'ivf_search_cnt.bucket', + 'load_latency_sum': 'load_latency.sum', + 'load_latency_count': 'load_latency.count', + 'load_latency_bucket': 'load_latency.bucket', 'process_cpu_seconds_total': 'process.cpu_seconds_total', 'process_max_fds': 'process.max_fds', 'process_open_fds': 'process.open_fds', @@ -212,13 +366,27 @@ 'process_start_time_seconds': 'process.start_time_seconds', 'process_virtual_memory_bytes': 'process.virtual_memory.bytes', 'process_virtual_memory_max_bytes': 'process.virtual_memory.max_bytes', - 'quant_compute_cnt': 'quant.compute_cnt', - 'queue_latency': 'queue.latency', - 'range_search_latency': 'range_search_latency', - 'raw_compute_cnt': 'raw_compute_cnt', - 're_search_cnt': 're_search_cnt', - 'search_latency': 'search.latency', - 'search_topk': 'search.topk', + 'quant_compute_cnt_sum': 'quant.compute_cnt.sum', + 'quant_compute_cnt_count': 'quant.compute_cnt.count', + 'quant_compute_cnt_bucket': 'quant.compute_cnt.bucket', + 'queue_latency_sum': 'queue.latency.sum', + 'queue_latency_count': 'queue.latency.count', + 'queue_latency_bucket': 'queue.latency.bucket', + 'range_search_latency_sum': 'range_search_latency.sum', + 'range_search_latency_count': 'range_search_latency.count', + 'range_search_latency_bucket': 'range_search_latency.bucket', + 'raw_compute_cnt_sum': 'raw_compute_cnt.sum', + 'raw_compute_cnt_count': 'raw_compute_cnt.count', + 'raw_compute_cnt_bucket': 'raw_compute_cnt.bucket', + 're_search_cnt_sum': 're_search_cnt.sum', + 're_search_cnt_count': 're_search_cnt.count', + 're_search_cnt_bucket': 're_search_cnt.bucket', + 'search_latency_sum': 'search.latency.sum', + 'search_latency_count': 'search.latency.count', + 'search_latency_bucket': 'search.latency.bucket', + 'search_topk_sum': 'search.topk.sum', + 'search_topk_count': 'search.topk.count', + 'search_topk_bucket': 'search.topk.bucket', } RENAME_LABELS_MAP = { diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 8ac12317f86ad..d9ca136437f30 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,8 +1,12 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus.cgo.cgo_duration_seconds,gauge,,second,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds,gauge,,second,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.cgo_duration_seconds.sum,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_duration_seconds.count,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_duration_seconds.bucket,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds.count,count,,second,,Duration of cgo call in queue.,0,milvus,, +milvus.cgo.cgo_queue_duration_seconds.bucket,count,,second,,Duration of cgo call in queue.,0,milvus,, milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, @@ -16,13 +20,17 @@ milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,, milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, -milvus.datacoord.task_execute_max_latency,gauge,,millisecond,,latency of task execute operation,0,milvus,, +milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,latency of task execute operation,0,milvus,, +milvus.datacoord.task_execute_max_latency.count,count,,millisecond,,latency of task execute operation,0,milvus,, +milvus.datacoord.task_execute_max_latency.bucket,count,,millisecond,,latency of task execute operation,0,milvus,, milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.datanode.encode_buffer_latency,gauge,,millisecond,,latency of encode buffer data,0,milvus,, +milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,latency of encode buffer data,0,milvus,, +milvus.datanode.encode_buffer_latency.count,count,,millisecond,,latency of encode buffer data,0,milvus,, +milvus.datanode.encode_buffer_latency.bucket,count,,millisecond,,latency of encode buffer data,0,milvus,, milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, @@ -30,61 +38,113 @@ milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milv milvus.datanode.flushed_data.size,count,,byte,,byte size of data flushed to storage,0,milvus,, milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus.datanode.save_latency,gauge,,millisecond,,latency of saving flush data to storage,0,milvus,, -milvus.flushed_segment_file_num,gauge,,,,the num of files for flushed segment,0,milvus,, -milvus.indexnode.build_index_latency,gauge,,millisecond,,latency of build index for segment,0,milvus,, -milvus.indexnode.encode_index_latency,gauge,,millisecond,,latency of encoding the index file,0,milvus,, +milvus.datanode.save_latency.sum,count,,millisecond,,latency of saving flush data to storage,0,milvus,, +milvus.datanode.save_latency.count,count,,millisecond,,latency of saving flush data to storage,0,milvus,, +milvus.datanode.save_latency.bucket,count,,millisecond,,latency of saving flush data to storage,0,milvus,, +milvus.flushed_segment_file_num.sum,count,,,,the num of files for flushed segment,0,milvus,, +milvus.flushed_segment_file_num.count,count,,,,the num of files for flushed segment,0,milvus,, +milvus.flushed_segment_file_num.bucket,count,,,,the num of files for flushed segment,0,milvus,, +milvus.indexnode.build_index_latency.sum,count,,millisecond,,latency of build index for segment,0,milvus,, +milvus.indexnode.build_index_latency.count,count,,millisecond,,latency of build index for segment,0,milvus,, +milvus.indexnode.build_index_latency.bucket,count,,millisecond,,latency of build index for segment,0,milvus,, +milvus.indexnode.encode_index_latency.sum,count,,millisecond,,latency of encoding the index file,0,milvus,, +milvus.indexnode.encode_index_latency.count,count,,millisecond,,latency of encoding the index file,0,milvus,, +milvus.indexnode.encode_index_latency.bucket,count,,millisecond,,latency of encoding the index file,0,milvus,, milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, -milvus.indexnode.index.task_latency_in_queue,gauge,,millisecond,,latency of index task in queue,0,milvus,, -milvus.indexnode.knowhere_build_index_latency,gauge,,millisecond,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.save_index_latency,gauge,,millisecond,,latency of saving the index file,0,milvus,, -milvus.meta.kv_size,gauge,,,,kv size stats,0,milvus,, +milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,latency of index task in queue,0,milvus,, +milvus.indexnode.index.task_latency_in_queue.count,count,,millisecond,,latency of index task in queue,0,milvus,, +milvus.indexnode.index.task_latency_in_queue.bucket,count,,millisecond,,latency of index task in queue,0,milvus,, +milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.knowhere_build_index_latency.count,count,,millisecond,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.knowhere_build_index_latency.bucket,count,,millisecond,,latency of building the index by knowhere,0,milvus,, +milvus.indexnode.save_index_latency.sum,count,,millisecond,,latency of saving the index file,0,milvus,, +milvus.indexnode.save_index_latency.count,count,,millisecond,,latency of saving the index file,0,milvus,, +milvus.indexnode.save_index_latency.bucket,count,,millisecond,,latency of saving the index file,0,milvus,, +milvus.meta.kv_size.sum,count,,,,kv size stats,0,milvus,, +milvus.meta.kv_size.count,count,,,,kv size stats,0,milvus,, +milvus.meta.kv_size.bucket,count,,,,kv size stats,0,milvus,, milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, -milvus.meta.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, +milvus.meta.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, +milvus.meta.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, +milvus.meta.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, -milvus.msgstream.request_latency,gauge,,millisecond,,request latency on the client side,0,milvus,, +milvus.msgstream.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, +milvus.msgstream.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, +milvus.msgstream.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus.proxy.apply.pk_latency,gauge,,millisecond,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.timestamp_latency,gauge,,millisecond,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.assign_segmentID_latency,gauge,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.pk_latency.count,count,,millisecond,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.pk_latency.bucket,count,,millisecond,,latency that apply primary key,0,milvus,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.apply.timestamp_latency.count,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.apply.timestamp_latency.bucket,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.assign_segmentID_latency.count,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, +milvus.proxy.assign_segmentID_latency.bucket,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, -milvus.proxy.cache.update_latency,gauge,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.cache.update_latency.count,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, +milvus.proxy.cache.update_latency.bucket,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus.proxy.mutation_send_latency,gauge,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.mutation_send_latency.count,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, +milvus.proxy.mutation_send_latency.bucket,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.req.in_queue_latency,gauge,,millisecond,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.latency,gauge,,millisecond,,latency of each request,0,milvus,, +milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.in_queue_latency.count,count,,millisecond,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.in_queue_latency.bucket,count,,millisecond,,latency which request waits in the queue,0,milvus,, +milvus.proxy.req.latency.sum,count,,millisecond,,latency of each request,0,milvus,, +milvus.proxy.req.latency.count,count,,millisecond,,latency of each request,0,milvus,, +milvus.proxy.req.latency.bucket,count,,millisecond,,latency of each request,0,milvus,, milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus.proxy.sq.decode_result_latency,gauge,,millisecond,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency,gauge,,millisecond,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.wait_result_latency,gauge,,millisecond,,latency that proxy waits for the result,0,milvus,, -milvus.proxy.sync_segment_request_length,gauge,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.decode_result_latency.count,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.decode_result_latency.bucket,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency.count,count,,millisecond,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,millisecond,,latency that proxy reduces search result,0,milvus,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sq.wait_result_latency.count,count,,millisecond,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sq.wait_result_latency.bucket,count,,millisecond,,latency that proxy waits for the result,0,milvus,, +milvus.proxy.sync_segment_request_length.sum,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.proxy.sync_segment_request_length.count,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, +milvus.proxy.sync_segment_request_length.bucket,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, -milvus.querycoord.load.latency,gauge,,millisecond,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.latency.sum,count,,millisecond,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.latency.count,count,,millisecond,,latency of load the entire collection,0,milvus,, +milvus.querycoord.load.latency.bucket,count,,millisecond,,latency of load the entire collection,0,milvus,, milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus.querycoord.release.latency,gauge,,millisecond,,latency of release request,0,milvus,, +milvus.querycoord.release.latency.sum,count,,millisecond,,latency of release request,0,milvus,, +milvus.querycoord.release.latency.count,count,,millisecond,,latency of release request,0,milvus,, +milvus.querycoord.release.latency.bucket,count,,millisecond,,latency of release request,0,milvus,, milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus.querynode.apply_bf_latency,gauge,,millisecond,,apply bf cost in ms,0,milvus,, +milvus.querynode.apply_bf_latency.sum,count,,millisecond,,apply bf cost in ms,0,milvus,, +milvus.querynode.apply_bf_latency.count,count,,millisecond,,apply bf cost in ms,0,milvus,, +milvus.querynode.apply_bf_latency.bucket,count,,millisecond,,apply bf cost in ms,0,milvus,, milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.querynode.disk.cache.evict.bytes,count,,byte,,number of bytes evicted from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.duration,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration,gauge,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration.count,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.global_duration.bucket,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, milvus.querynode.disk.cache.load.bytes,count,,byte,,number of bytes loaded from disk cache,0,milvus,, milvus.querynode.disk.cache.load.duration,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration,gauge,,millisecond,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration.count,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.global_duration.bucket,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, @@ -92,44 +152,88 @@ milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/q milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.querynode.forward_delete_latency,gauge,,millisecond,,forward delete cost in ms,0,milvus,, -milvus.querynode.load.index_latency,gauge,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.forward_delete_latency.sum,count,,millisecond,,forward delete cost in ms,0,milvus,, +milvus.querynode.forward_delete_latency.count,count,,millisecond,,forward delete cost in ms,0,milvus,, +milvus.querynode.forward_delete_latency.bucket,count,,millisecond,,forward delete cost in ms,0,milvus,, +milvus.querynode.load.index_latency.sum,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.load.index_latency.count,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, +milvus.querynode.load.index_latency.bucket,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus.querynode.load.segment.latency,gauge,,millisecond,,latency of load per segment,0,milvus,, +milvus.querynode.load.segment.latency.sum,count,,millisecond,,latency of load per segment,0,milvus,, +milvus.querynode.load.segment.latency.count,count,,millisecond,,latency of load per segment,0,milvus,, +milvus.querynode.load.segment.latency.bucket,count,,millisecond,,latency of load per segment,0,milvus,, milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus.querynode.process_insert_or_delete_latency,gauge,,millisecond,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.process_insert_or_delete_latency.count,count,,millisecond,,process insert or delete cost in ms,0,milvus,, +milvus.querynode.process_insert_or_delete_latency.bucket,count,,millisecond,,process insert or delete cost in ms,0,milvus,, milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, -milvus.querynode.search.group.nq,gauge,,,,the number of queries of each grouped search task,0,milvus,, -milvus.querynode.search.group.size,gauge,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.querynode.search.group.topk,gauge,,,,the topK of each grouped search task,0,milvus,, -milvus.querynode.search.nq,gauge,,,,the number of queries of each search task,0,milvus,, -milvus.querynode.search.topk,gauge,,,,the top of each search task,0,milvus,, +milvus.querynode.search.group.nq.sum,count,,,,the number of queries of each grouped search task,0,milvus,, +milvus.querynode.search.group.nq.count,count,,,,the number of queries of each grouped search task,0,milvus,, +milvus.querynode.search.group.nq.bucket,count,,,,the number of queries of each grouped search task,0,milvus,, +milvus.querynode.search.group.size.sum,count,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.querynode.search.group.size.count,count,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.querynode.search.group.size.bucket,count,,,,the number of tasks of each grouped search task,0,milvus,, +milvus.querynode.search.group.topk.sum,count,,,,the topK of each grouped search task,0,milvus,, +milvus.querynode.search.group.topk.count,count,,,,the topK of each grouped search task,0,milvus,, +milvus.querynode.search.group.topk.bucket,count,,,,the topK of each grouped search task,0,milvus,, +milvus.querynode.search.nq.sum,count,,,,the number of queries of each search task,0,milvus,, +milvus.querynode.search.nq.count,count,,,,the number of queries of each search task,0,milvus,, +milvus.querynode.search.nq.bucket,count,,,,the number of queries of each search task,0,milvus,, +milvus.querynode.search.topk.sum,count,,,,the top of each search task,0,milvus,, +milvus.querynode.search.topk.count,count,,,,the top of each search task,0,milvus,, +milvus.querynode.search.topk.bucket,count,,,,the top of each search task,0,milvus,, milvus.querynode.segment.access.duration,count,,millisecond,,total time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration,gauge,,millisecond,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration.count,count,,millisecond,,global time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.global_duration.bucket,count,,millisecond,,global time cost of accessing segments,0,milvus,, milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, milvus.querynode.segment.access.wait_cache.duration,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration,gauge,,millisecond,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration.count,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, -milvus.querynode.segment.latency_per_vector,gauge,,millisecond,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.latency_per_vector.count,count,,millisecond,,one vector's search latency per segment,0,milvus,, +milvus.querynode.segment.latency_per_vector.bucket,count,,millisecond,,one vector's search latency per segment,0,milvus,, milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus.querynode.sq.core_latency,gauge,,millisecond,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.queue.latency,gauge,,millisecond,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency,gauge,,millisecond,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.reduce_latency,gauge,,millisecond,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.core_latency.sum,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.core_latency.count,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.core_latency.bucket,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, +milvus.querynode.sq.queue.latency.sum,count,,millisecond,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.latency.count,count,,millisecond,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.latency.bucket,count,,millisecond,,latency of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency.count,count,,millisecond,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.queue.user_latency.bucket,count,,millisecond,,latency per user of search or query in queue,0,milvus,, +milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.reduce_latency.count,count,,millisecond,,latency of reduce search or query result,0,milvus,, +milvus.querynode.sq.reduce_latency.bucket,count,,millisecond,,latency of reduce search or query result,0,milvus,, milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, -milvus.querynode.sq.req.latency,gauge,,millisecond,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.segment_latency,gauge,,millisecond,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency,gauge,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.sq.req.latency.sum,count,,millisecond,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.req.latency.count,count,,millisecond,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.req.latency.bucket,count,,millisecond,,latency of Search or query requests,0,milvus,, +milvus.querynode.sq.segment_latency.sum,count,,millisecond,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.segment_latency.count,count,,millisecond,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.segment_latency.bucket,count,,millisecond,,latency of search or query per segment,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency.count,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, +milvus.querynode.sq.wait_tsafe_latency.bucket,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, -milvus.querynode.watch_dml_channel_latency,gauge,,millisecond,,latency of watch dml channel,0,milvus,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,latency of watch dml channel,0,milvus,, +milvus.querynode.watch_dml_channel_latency.count,count,,millisecond,,latency of watch dml channel,0,milvus,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,millisecond,,latency of watch dml channel,0,milvus,, milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency,gauge,,millisecond,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue,gauge,,millisecond,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency.count,count,,millisecond,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency.bucket,count,,millisecond,,latency of each DDL operations,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue.count,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, +milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, @@ -141,26 +245,54 @@ milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus.rootcoord.sync_timetick_latency,gauge,,millisecond,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.sync_timetick_latency.count,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, +milvus.rootcoord.sync_timetick_latency.bucket,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,, milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus.bf_search_cnt,gauge,,,,number of bf search per request,0,milvus,, -milvus.bitset_ratio,gauge,,,,bitset ratio,0,milvus,, -milvus.build_latency,gauge,,,,index build latency (s),0,milvus,, -milvus.cache_hit_cnt,gauge,,,,cache hit cnt per request,0,milvus,, -milvus.diskann.range_search_iters,gauge,,,,DISKANN range search iterations,0,milvus,, -milvus.diskann.search_hops,gauge,,,,DISKANN search hops,0,milvus,, -milvus.diskann_bitset_ratio,gauge,,,,DISKANN bitset ratio for search and range search,0,milvus,, -milvus.exec_latency,gauge,,,,execute latency per request,0,milvus,, -milvus.filter.connectivity_ratio,gauge,,,,avg connectivity ratio set under filtering per request,0,milvus,, -milvus.filter.mv.activated_fields_cnt,gauge,,,,avg mv activated fields per request,0,milvus,, -milvus.filter.mv.change_base_cnt,gauge,,,,mv change base cnt per request,0,milvus,, -milvus.filter.mv.only_cnt,gauge,,,,mv only cnt per request,0,milvus,, -milvus.filter.mv.supplement_ep_bool_cnt,gauge,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, +milvus.bf_search_cnt.sum,count,,,,number of bf search per request,0,milvus,, +milvus.bf_search_cnt.count,count,,,,number of bf search per request,0,milvus,, +milvus.bf_search_cnt.bucket,count,,,,number of bf search per request,0,milvus,, +milvus.bitset_ratio.sum,count,,,,bitset ratio,0,milvus,, +milvus.bitset_ratio.count,count,,,,bitset ratio,0,milvus,, +milvus.bitset_ratio.bucket,count,,,,bitset ratio,0,milvus,, +milvus.build_latency.sum,count,,,,index build latency (s),0,milvus,, +milvus.build_latency.count,count,,,,index build latency (s),0,milvus,, +milvus.build_latency.bucket,count,,,,index build latency (s),0,milvus,, +milvus.cache_hit_cnt.sum,count,,,,cache hit cnt per request,0,milvus,, +milvus.cache_hit_cnt.count,count,,,,cache hit cnt per request,0,milvus,, +milvus.cache_hit_cnt.bucket,count,,,,cache hit cnt per request,0,milvus,, +milvus.diskann.range_search_iters.sum,count,,,,DISKANN range search iterations,0,milvus,, +milvus.diskann.range_search_iters.count,count,,,,DISKANN range search iterations,0,milvus,, +milvus.diskann.range_search_iters.bucket,count,,,,DISKANN range search iterations,0,milvus,, +milvus.diskann.search_hops.sum,count,,,,DISKANN search hops,0,milvus,, +milvus.diskann.search_hops.count,count,,,,DISKANN search hops,0,milvus,, +milvus.diskann.search_hops.bucket,count,,,,DISKANN search hops,0,milvus,, +milvus.diskann_bitset_ratio.sum,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, +milvus.diskann_bitset_ratio.count,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, +milvus.diskann_bitset_ratio.bucket,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, +milvus.exec_latency.sum,count,,,,execute latency per request,0,milvus,, +milvus.exec_latency.count,count,,,,execute latency per request,0,milvus,, +milvus.exec_latency.bucket,count,,,,execute latency per request,0,milvus,, +milvus.filter.connectivity_ratio.sum,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, +milvus.filter.connectivity_ratio.count,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, +milvus.filter.connectivity_ratio.bucket,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, +milvus.filter.mv.activated_fields_cnt.sum,count,,,,avg mv activated fields per request,0,milvus,, +milvus.filter.mv.activated_fields_cnt.count,count,,,,avg mv activated fields per request,0,milvus,, +milvus.filter.mv.activated_fields_cnt.bucket,count,,,,avg mv activated fields per request,0,milvus,, +milvus.filter.mv.change_base_cnt.sum,count,,,,mv change base cnt per request,0,milvus,, +milvus.filter.mv.change_base_cnt.count,count,,,,mv change base cnt per request,0,milvus,, +milvus.filter.mv.change_base_cnt.bucket,count,,,,mv change base cnt per request,0,milvus,, +milvus.filter.mv.only_cnt.sum,count,,,,mv only cnt per request,0,milvus,, +milvus.filter.mv.only_cnt.count,count,,,,mv only cnt per request,0,milvus,, +milvus.filter.mv.only_cnt.bucket,count,,,,mv only cnt per request,0,milvus,, +milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, +milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, +milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, milvus.go.gc_duration_seconds,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,, @@ -188,19 +320,41 @@ milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stac milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,, milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,, milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,, -milvus.graph_search_cnt,gauge,,,,number of graph search per request,0,milvus,, -milvus.hnsw.bitset_ratio,gauge,,,,HNSW bitset ratio for search and range search,0,milvus,, -milvus.hnsw.search_hops,gauge,,,,HNSW search hops in layer 0,0,milvus,, -milvus.internal.core_search_latency,gauge,,,,[cpp]latency(us) of search on segment,0,milvus,, -milvus.internal.mmap.allocated_space_bytes,gauge,,,,[cpp]mmap allocated space stats,0,milvus,, +milvus.graph_search_cnt.sum,count,,,,number of graph search per request,0,milvus,, +milvus.graph_search_cnt.count,count,,,,number of graph search per request,0,milvus,, +milvus.graph_search_cnt.bucket,count,,,,number of graph search per request,0,milvus,, +milvus.hnsw.bitset_ratio.sum,count,,,,HNSW bitset ratio for search and range search,0,milvus,, +milvus.hnsw.bitset_ratio.count,count,,,,HNSW bitset ratio for search and range search,0,milvus,, +milvus.hnsw.bitset_ratio.bucket,count,,,,HNSW bitset ratio for search and range search,0,milvus,, +milvus.hnsw.search_hops.sum,count,,,,HNSW search hops in layer 0,0,milvus,, +milvus.hnsw.search_hops.count,count,,,,HNSW search hops in layer 0,0,milvus,, +milvus.hnsw.search_hops.bucket,count,,,,HNSW search hops in layer 0,0,milvus,, +milvus.internal.core_search_latency.sum,count,,,,[cpp]latency(us) of search on segment,0,milvus,, +milvus.internal.core_search_latency.count,count,,,,[cpp]latency(us) of search on segment,0,milvus,, +milvus.internal.core_search_latency.bucket,count,,,,[cpp]latency(us) of search on segment,0,milvus,, +milvus.internal.mmap.allocated_space_bytes.sum,count,,,,[cpp]mmap allocated space stats,0,milvus,, +milvus.internal.mmap.allocated_space_bytes.count,count,,,,[cpp]mmap allocated space stats,0,milvus,, +milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,[cpp]mmap allocated space stats,0,milvus,, milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,, -milvus.internal.storage.kv_size,gauge,,,,[cpp]kv size stats,0,milvus,, -milvus.internal.storage.load_duration,gauge,,,,[cpp]durations of load segment,0,milvus,, +milvus.internal.storage.kv_size.sum,count,,,,[cpp]kv size stats,0,milvus,, +milvus.internal.storage.kv_size.count,count,,,,[cpp]kv size stats,0,milvus,, +milvus.internal.storage.kv_size.bucket,count,,,,[cpp]kv size stats,0,milvus,, +milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segment,0,milvus,, +milvus.internal.storage.load_duration.count,count,,,,[cpp]durations of load segment,0,milvus,, +milvus.internal.storage.load_duration.bucket,count,,,,[cpp]durations of load segment,0,milvus,, milvus.internal.storage.op_count,count,,,,[cpp]count of persistent data operation,0,milvus,, -milvus.internal.storage.request_latency,gauge,,,,[cpp]request latency(ms) on the client side,0,milvus,, -milvus.io_cnt,gauge,,,,io cnt per request,0,milvus,, -milvus.ivf_search_cnt,gauge,,,,number of ivf search per request,0,milvus,, -milvus.load_latency,gauge,,,,index load latency (ms),0,milvus,, +milvus.internal.storage.request_latency.sum,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, +milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, +milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, +milvus.io_cnt.sum,count,,,,io cnt per request,0,milvus,, +milvus.io_cnt.count,count,,,,io cnt per request,0,milvus,, +milvus.io_cnt.bucket,count,,,,io cnt per request,0,milvus,, +milvus.ivf_search_cnt.sum,count,,,,number of ivf search per request,0,milvus,, +milvus.ivf_search_cnt.count,count,,,,number of ivf search per request,0,milvus,, +milvus.ivf_search_cnt.bucket,count,,,,number of ivf search per request,0,milvus,, +milvus.load_latency.sum,count,,,,index load latency (ms),0,milvus,, +milvus.load_latency.count,count,,,,index load latency (ms),0,milvus,, +milvus.load_latency.bucket,count,,,,index load latency (ms),0,milvus,, milvus.process.cpu_seconds_total,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, @@ -208,10 +362,24 @@ milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,m milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,, milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,, milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,, -milvus.quant.compute_cnt,gauge,,,,quant compute cnt per request,0,milvus,, -milvus.queue.latency,gauge,,,,queue latency per request,0,milvus,, -milvus.range_search_latency,gauge,,,,range search latency (ms),0,milvus,, -milvus.raw_compute_cnt,gauge,,,,raw compute cnt per request,0,milvus,, -milvus.re_search_cnt,gauge,,,,number of fallback search per request,0,milvus,, -milvus.search.latency,gauge,,,,search latency (ms),0,milvus,, -milvus.search.topk,gauge,,,,search topk,0,milvus,, +milvus.quant.compute_cnt.sum,count,,,,quant compute cnt per request,0,milvus,, +milvus.quant.compute_cnt.count,count,,,,quant compute cnt per request,0,milvus,, +milvus.quant.compute_cnt.bucket,count,,,,quant compute cnt per request,0,milvus,, +milvus.queue.latency.sum,count,,,,queue latency per request,0,milvus,, +milvus.queue.latency.count,count,,,,queue latency per request,0,milvus,, +milvus.queue.latency.bucket,count,,,,queue latency per request,0,milvus,, +milvus.range_search_latency.sum,count,,,,range search latency (ms),0,milvus,, +milvus.range_search_latency.count,count,,,,range search latency (ms),0,milvus,, +milvus.range_search_latency.bucket,count,,,,range search latency (ms),0,milvus,, +milvus.raw_compute_cnt.sum,count,,,,raw compute cnt per request,0,milvus,, +milvus.raw_compute_cnt.count,count,,,,raw compute cnt per request,0,milvus,, +milvus.raw_compute_cnt.bucket,count,,,,raw compute cnt per request,0,milvus,, +milvus.re_search_cnt.sum,count,,,,number of fallback search per request,0,milvus,, +milvus.re_search_cnt.count,count,,,,number of fallback search per request,0,milvus,, +milvus.re_search_cnt.bucket,count,,,,number of fallback search per request,0,milvus,, +milvus.search.latency.sum,count,,,,search latency (ms),0,milvus,, +milvus.search.latency.count,count,,,,search latency (ms),0,milvus,, +milvus.search.latency.bucket,count,,,,search latency (ms),0,milvus,, +milvus.search.topk.sum,count,,,,search topk,0,milvus,, +milvus.search.topk.count,count,,,,search topk,0,milvus,, +milvus.search.topk.bucket,count,,,,search topk,0,milvus,, From f4141461d9c9898ff9a4181e30231da824cfc5e8 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Thu, 28 Nov 2024 12:18:31 +0100 Subject: [PATCH 17/31] Fix metrics mapping and divide test metrics --- milvus/datadog_checks/milvus/check.py | 2 +- .../milvus/data/conf.yaml.example | 2 +- milvus/datadog_checks/milvus/metrics.py | 404 +++++----------- milvus/metadata.csv | 4 +- milvus/tests/common.py | 455 ++++++++++++++---- milvus/tests/test_unit.py | 10 +- 6 files changed, 473 insertions(+), 404 deletions(-) diff --git a/milvus/datadog_checks/milvus/check.py b/milvus/datadog_checks/milvus/check.py index 707f6ae7cf88f..d1be0f56ca38e 100644 --- a/milvus/datadog_checks/milvus/check.py +++ b/milvus/datadog_checks/milvus/check.py @@ -2,8 +2,8 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.milvus.metrics import METRIC_MAP, RENAME_LABELS_MAP from datadog_checks.base import OpenMetricsBaseCheckV2 +from datadog_checks.milvus.metrics import METRIC_MAP, RENAME_LABELS_MAP class MilvusCheck(OpenMetricsBaseCheckV2): diff --git a/milvus/datadog_checks/milvus/data/conf.yaml.example b/milvus/datadog_checks/milvus/data/conf.yaml.example index b275903aba352..30d8e0cf300cb 100644 --- a/milvus/datadog_checks/milvus/data/conf.yaml.example +++ b/milvus/datadog_checks/milvus/data/conf.yaml.example @@ -180,7 +180,7 @@ instances: ## @param collect_histogram_buckets - boolean - optional - default: true ## Whether or not to send histogram buckets. # - # collect_histogram_buckets: true + # collect_histogram_buckets: false ## @param non_cumulative_histogram_buckets - boolean - optional - default: false ## Whether or not histogram buckets are non-cumulative and to come with a `lower_bound` tag. diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py index 8ba8a77036d3c..6f70c7e42ab59 100644 --- a/milvus/datadog_checks/milvus/metrics.py +++ b/milvus/datadog_checks/milvus/metrics.py @@ -5,12 +5,8 @@ METRIC_MAP = { 'milvus_build_info': 'build_info', 'milvus_cgo_active_future_total': 'cgo.active_future_total', - 'milvus_cgo_cgo_duration_seconds_sum': 'cgo.cgo_duration_seconds.sum', - 'milvus_cgo_cgo_duration_seconds_count': 'cgo.cgo_duration_seconds.count', - 'milvus_cgo_cgo_duration_seconds_bucket': 'cgo.cgo_duration_seconds.bucket', - 'milvus_cgo_cgo_queue_duration_seconds_sum': 'cgo.cgo_queue_duration_seconds.sum', - 'milvus_cgo_cgo_queue_duration_seconds_count': 'cgo.cgo_queue_duration_seconds.count', - 'milvus_cgo_cgo_queue_duration_seconds_bucket': 'cgo.cgo_queue_duration_seconds.bucket', + 'milvus_cgo_cgo_duration_seconds': 'cgo.cgo_duration_seconds', + 'milvus_cgo_cgo_queue_duration_seconds': 'cgo.cgo_queue_duration_seconds', 'milvus_cgo_running_cgo_call_total': 'cgo.running_cgo_call_total', 'milvus_datacoord_channel_checkpoint_unix_seconds': 'datacoord.channel_checkpoint_unix_seconds', 'milvus_datacoord_collection_num': 'datacoord.collection_num', @@ -18,230 +14,130 @@ 'milvus_datacoord_datanode_num': 'datacoord.datanode_num', 'milvus_datacoord_import_tasks': 'datacoord.import_tasks', 'milvus_datacoord_index_node_num': 'datacoord.index.node_num', - 'milvus_datacoord_index_req_count': 'datacoord.index.req_count', - 'milvus_datacoord_index_task_count': 'datacoord.index.task_count', + 'milvus_datacoord_index_req_count': 'datacoord.index.req', + 'milvus_datacoord_index_task_count': 'datacoord.index.task', 'milvus_datacoord_segment_num': 'datacoord.segment_num', 'milvus_datacoord_stored_binlog_size': 'datacoord.stored.binlog_size', 'milvus_datacoord_stored_index_files_size': 'datacoord.stored.index_files_size', 'milvus_datacoord_stored_rows_num': 'datacoord.stored.rows_num', - 'milvus_datacoord_task_execute_max_latency_sum': 'datacoord.task_execute_max_latency.sum', - 'milvus_datacoord_task_execute_max_latency_count': 'datacoord.task_execute_max_latency.count', - 'milvus_datacoord_task_execute_max_latency_bucket': 'datacoord.task_execute_max_latency.bucket', + 'milvus_datacoord_task_execute_max_latency': 'datacoord.task_execute_max_latency', 'milvus_datacoord_watched_dml_chanel_num': 'datacoord.watched_dml_chanel_num', - 'milvus_datanode_autoflush_buffer_op_count': 'datanode.autoflush_buffer_op_count', - 'milvus_datanode_consume_bytes_count': 'datanode.consume.bytes_count', - 'milvus_datanode_consume_msg_count': 'datanode.consume.msg_count', + 'milvus_datanode_autoflush_buffer_op_count': 'datanode.autoflush_buffer_op', + 'milvus_datanode_consume_bytes_count': 'datanode.consume.bytes', + 'milvus_datanode_consume_msg_count': 'datanode.consume.msg', 'milvus_datanode_consume_tt_lag_ms': 'datanode.consume.tt_lag_ms', - 'milvus_datanode_encode_buffer_latency_sum': 'datanode.encode_buffer_latency.sum', - 'milvus_datanode_encode_buffer_latency_count': 'datanode.encode_buffer_latency.count', - 'milvus_datanode_encode_buffer_latency_bucket': 'datanode.encode_buffer_latency.bucket', + 'milvus_datanode_encode_buffer_latency': 'datanode.encode_buffer_latency', 'milvus_datanode_flowgraph_num': 'datanode.flowgraph_num', - 'milvus_datanode_flush_buffer_op_count': 'datanode.flush.buffer_op_count', - 'milvus_datanode_flush_req_count': 'datanode.flush.req_count', + 'milvus_datanode_flush_buffer_op_count': 'datanode.flush.buffer_op', + 'milvus_datanode_flush_req_count': 'datanode.flush.req', 'milvus_datanode_flushed_data_rows': 'datanode.flushed_data.rows', 'milvus_datanode_flushed_data_size': 'datanode.flushed_data.size', 'milvus_datanode_msg_dispatcher_tt_lag_ms': 'datanode.msg.dispatcher_tt_lag_ms', - 'milvus_datanode_msg_rows_count': 'datanode.msg.rows_count', - 'milvus_datanode_save_latency_sum': 'datanode.save_latency.sum', - 'milvus_datanode_save_latency_count': 'datanode.save_latency.count', - 'milvus_datanode_save_latency_bucket': 'datanode.save_latency.bucket', - 'milvus_flushed_segment_file_num_sum': 'flushed_segment_file_num.sum', - 'milvus_flushed_segment_file_num_count': 'flushed_segment_file_num.count', - 'milvus_flushed_segment_file_num_bucket': 'flushed_segment_file_num.bucket', - 'milvus_indexnode_build_index_latency_sum': 'indexnode.build_index_latency.sum', - 'milvus_indexnode_build_index_latency_count': 'indexnode.build_index_latency.count', - 'milvus_indexnode_build_index_latency_bucket': 'indexnode.build_index_latency.bucket', - 'milvus_indexnode_encode_index_latency_sum': 'indexnode.encode_index_latency.sum', - 'milvus_indexnode_encode_index_latency_count': 'indexnode.encode_index_latency.count', - 'milvus_indexnode_encode_index_latency_bucket': 'indexnode.encode_index_latency.bucket', - 'milvus_indexnode_index_task_count': 'indexnode.index.task_count', - 'milvus_indexnode_index_task_latency_in_queue_sum': 'indexnode.index.task_latency_in_queue.sum', - 'milvus_indexnode_index_task_latency_in_queue_count': 'indexnode.index.task_latency_in_queue.count', - 'milvus_indexnode_index_task_latency_in_queue_bucket': 'indexnode.index.task_latency_in_queue.bucket', - 'milvus_indexnode_knowhere_build_index_latency_sum': 'indexnode.knowhere_build_index_latency.sum', - 'milvus_indexnode_knowhere_build_index_latency_count': 'indexnode.knowhere_build_index_latency.count', - 'milvus_indexnode_knowhere_build_index_latency_bucket': 'indexnode.knowhere_build_index_latency.bucket', - 'milvus_indexnode_save_index_latency_sum': 'indexnode.save_index_latency.sum', - 'milvus_indexnode_save_index_latency_count': 'indexnode.save_index_latency.count', - 'milvus_indexnode_save_index_latency_bucket': 'indexnode.save_index_latency.bucket', - 'milvus_meta_kv_size_sum': 'meta.kv_size.sum', - 'milvus_meta_kv_size_count': 'meta.kv_size.count', - 'milvus_meta_kv_size_bucket': 'meta.kv_size.bucket', - 'milvus_meta_op_count': 'meta.op_count', - 'milvus_meta_request_latency_sum': 'meta.request_latency.sum', - 'milvus_meta_request_latency_count': 'meta.request_latency.count', - 'milvus_meta_request_latency_bucket': 'meta.request_latency.bucket', + 'milvus_datanode_msg_rows_count': 'datanode.msg.rows', + 'milvus_datanode_save_latency': 'datanode.save_latency', + 'milvus_flushed_segment_file_num': 'flushed_segment_file_num', + 'milvus_indexnode_build_index_latency': 'indexnode.build_index_latency', + 'milvus_indexnode_encode_index_latency': 'indexnode.encode_index_latency', + 'milvus_indexnode_index_task_count': 'indexnode.index.task', + 'milvus_indexnode_index_task_latency_in_queue': 'indexnode.index.task_latency_in_queue', + 'milvus_indexnode_knowhere_build_index_latency': 'indexnode.knowhere_build_index_latency', + 'milvus_indexnode_save_index_latency': 'indexnode.save_index_latency', + 'milvus_meta_kv_size': 'meta.kv_size', + 'milvus_meta_op_count': 'meta.op', + 'milvus_meta_request_latency': 'meta.request_latency', 'milvus_msg_queue_consumer_num': 'msg_queue_consumer_num', - 'milvus_msgstream_op_count': 'msgstream.op_count', - 'milvus_msgstream_request_latency_sum': 'msgstream.request_latency.sum', - 'milvus_msgstream_request_latency_count': 'msgstream.request_latency.count', - 'milvus_msgstream_request_latency_bucket': 'msgstream.request_latency.bucket', + 'milvus_msgstream_op_count': 'msgstream.op', + 'milvus_msgstream_request_latency': 'msgstream.request_latency', 'milvus_num_node': 'num_node', - 'milvus_proxy_apply_pk_latency_sum': 'proxy.apply.pk_latency.sum', - 'milvus_proxy_apply_pk_latency_count': 'proxy.apply.pk_latency.count', - 'milvus_proxy_apply_pk_latency_bucket': 'proxy.apply.pk_latency.bucket', - 'milvus_proxy_apply_timestamp_latency_sum': 'proxy.apply.timestamp_latency.sum', - 'milvus_proxy_apply_timestamp_latency_count': 'proxy.apply.timestamp_latency.count', - 'milvus_proxy_apply_timestamp_latency_bucket': 'proxy.apply.timestamp_latency.bucket', - 'milvus_proxy_assign_segmentID_latency_sum': 'proxy.assign_segmentID_latency.sum', - 'milvus_proxy_assign_segmentID_latency_count': 'proxy.assign_segmentID_latency.count', - 'milvus_proxy_assign_segmentID_latency_bucket': 'proxy.assign_segmentID_latency.bucket', - 'milvus_proxy_cache_hit_count': 'proxy.cache.hit_count', - 'milvus_proxy_cache_update_latency_sum': 'proxy.cache.update_latency.sum', - 'milvus_proxy_cache_update_latency_count': 'proxy.cache.update_latency.count', - 'milvus_proxy_cache_update_latency_bucket': 'proxy.cache.update_latency.bucket', - 'milvus_proxy_delete_vectors_count': 'proxy.delete_vectors_count', + 'milvus_proxy_apply_pk_latency': 'proxy.apply.pk_latency', + 'milvus_proxy_apply_timestamp_latency': 'proxy.apply.timestamp_latency', + 'milvus_proxy_assign_segmentID_latency': 'proxy.assign_segmentID_latency', + 'milvus_proxy_cache_hit_count': 'proxy.cache.hit', + 'milvus_proxy_cache_update_latency': 'proxy.cache.update_latency', + 'milvus_proxy_delete_vectors_count': 'proxy.delete_vectors', 'milvus_proxy_msgstream_obj_num': 'proxy.msgstream_obj_num', - 'milvus_proxy_mutation_send_latency_sum': 'proxy.mutation_send_latency.sum', - 'milvus_proxy_mutation_send_latency_count': 'proxy.mutation_send_latency.count', - 'milvus_proxy_mutation_send_latency_bucket': 'proxy.mutation_send_latency.bucket', - 'milvus_proxy_rate_limit_req_count': 'proxy.rate_limit_req_count', + 'milvus_proxy_mutation_send_latency': 'proxy.mutation_send_latency', + 'milvus_proxy_rate_limit_req_count': 'proxy.rate_limit_req', 'milvus_proxy_report_value': 'proxy.report_value', - 'milvus_proxy_req_count': 'proxy.req.count', - 'milvus_proxy_req_in_queue_latency_sum': 'proxy.req.in_queue_latency.sum', - 'milvus_proxy_req_in_queue_latency_count': 'proxy.req.in_queue_latency.count', - 'milvus_proxy_req_in_queue_latency_bucket': 'proxy.req.in_queue_latency.bucket', - 'milvus_proxy_req_latency_sum': 'proxy.req.latency.sum', - 'milvus_proxy_req_latency_count': 'proxy.req.latency.count', - 'milvus_proxy_req_latency_bucket': 'proxy.req.latency.bucket', - 'milvus_proxy_send_bytes_count': 'proxy.send_bytes_count', - 'milvus_proxy_sq_decode_result_latency_sum': 'proxy.sq.decode_result_latency.sum', - 'milvus_proxy_sq_decode_result_latency_count': 'proxy.sq.decode_result_latency.count', - 'milvus_proxy_sq_decode_result_latency_bucket': 'proxy.sq.decode_result_latency.bucket', - 'milvus_proxy_sq_reduce_result_latency_sum': 'proxy.sq.reduce_result_latency.sum', - 'milvus_proxy_sq_reduce_result_latency_count': 'proxy.sq.reduce_result_latency.count', - 'milvus_proxy_sq_reduce_result_latency_bucket': 'proxy.sq.reduce_result_latency.bucket', - 'milvus_proxy_sq_wait_result_latency_sum': 'proxy.sq.wait_result_latency.sum', - 'milvus_proxy_sq_wait_result_latency_count': 'proxy.sq.wait_result_latency.count', - 'milvus_proxy_sq_wait_result_latency_bucket': 'proxy.sq.wait_result_latency.bucket', - 'milvus_proxy_sync_segment_request_length_sum': 'proxy.sync_segment_request_length.sum', - 'milvus_proxy_sync_segment_request_length_count': 'proxy.sync_segment_request_length.count', - 'milvus_proxy_sync_segment_request_length_bucket': 'proxy.sync_segment_request_length.bucket', + 'milvus_proxy_req_count': 'proxy.req', + 'milvus_proxy_req_in_queue_latency': 'proxy.req.in_queue_latency', + 'milvus_proxy_req_latency': 'proxy.req.latency', + 'milvus_proxy_send_bytes_count': 'proxy.send_bytes', + 'milvus_proxy_sq_decode_result_latency': 'proxy.sq.decode_result_latency', + 'milvus_proxy_sq_reduce_result_latency': 'proxy.sq.reduce_result_latency', + 'milvus_proxy_sq_wait_result_latency': 'proxy.sq.wait_result_latency', + 'milvus_proxy_sync_segment_request_length': 'proxy.sync_segment_request_length', 'milvus_proxy_tt_lag_ms': 'proxy.tt_lag_ms', 'milvus_querycoord_collection_num': 'querycoord.collection_num', 'milvus_querycoord_current_target_checkpoint_unix_seconds': 'querycoord.current_target_checkpoint_unix_seconds', - 'milvus_querycoord_load_latency_sum': 'querycoord.load.latency.sum', - 'milvus_querycoord_load_latency_count': 'querycoord.load.latency.count', - 'milvus_querycoord_load_latency_bucket': 'querycoord.load.latency.bucket', - 'milvus_querycoord_load_req_count': 'querycoord.load.req_count', + 'milvus_querycoord_load_latency': 'querycoord.load.latency', + 'milvus_querycoord_load_req_count': 'querycoord.load.req', 'milvus_querycoord_partition_num': 'querycoord.partition_num', 'milvus_querycoord_querynode_num': 'querycoord.querynode_num', - 'milvus_querycoord_release_latency_sum': 'querycoord.release.latency.sum', - 'milvus_querycoord_release_latency_count': 'querycoord.release.latency.count', - 'milvus_querycoord_release_latency_bucket': 'querycoord.release.latency.bucket', - 'milvus_querycoord_release_req_count': 'querycoord.release.req_count', + 'milvus_querycoord_release_latency': 'querycoord.release.latency', + 'milvus_querycoord_release_req_count': 'querycoord.release.req', 'milvus_querycoord_task_num': 'querycoord_task_num', - 'milvus_querynode_apply_bf_latency_sum': 'querynode.apply_bf_latency.sum', - 'milvus_querynode_apply_bf_latency_count': 'querynode.apply_bf_latency.count', - 'milvus_querynode_apply_bf_latency_bucket': 'querynode.apply_bf_latency.bucket', + 'milvus_querynode_apply_bf_latency': 'querynode.apply_bf_latency', 'milvus_querynode_collection_num': 'querynode.collection_num', 'milvus_querynode_consume_bytes_counter': 'querynode.consume.bytes_counter', - 'milvus_querynode_consume_msg_count': 'querynode.consume.msg_count', + 'milvus_querynode_consume_msg_count': 'querynode.consume.msg', 'milvus_querynode_consume_tt_lag_ms': 'querynode.consume.tt_lag_ms', 'milvus_querynode_disk_cache_evict_bytes': 'querynode.disk.cache.evict.bytes', 'milvus_querynode_disk_cache_evict_duration': 'querynode.disk.cache.evict.duration', - 'milvus_querynode_disk_cache_evict_global_duration_sum': 'querynode.disk.cache.evict.global_duration.sum', - 'milvus_querynode_disk_cache_evict_global_duration_count': 'querynode.disk.cache.evict.global_duration.count', - 'milvus_querynode_disk_cache_evict_global_duration_bucket': 'querynode.disk.cache.evict.global_duration.bucket', - 'milvus_querynode_disk_cache_evict_total': 'querynode.disk.cache.evict.total', + 'milvus_querynode_disk_cache_evict_global_duration': 'querynode.disk.cache.evict.global_duration', + 'milvus_querynode_disk_cache_evict': 'querynode.disk.cache.evict', 'milvus_querynode_disk_cache_load_bytes': 'querynode.disk.cache.load.bytes', 'milvus_querynode_disk_cache_load_duration': 'querynode.disk.cache.load.duration', - 'milvus_querynode_disk_cache_load_global_duration_sum': 'querynode.disk.cache.load.global_duration.sum', - 'milvus_querynode_disk_cache_load_global_duration_count': 'querynode.disk.cache.load.global_duration.count', - 'milvus_querynode_disk_cache_load_global_duration_bucket': 'querynode.disk.cache.load.global_duration.bucket', - 'milvus_querynode_disk_cache_load_total': 'querynode.disk.cache.load.total', + 'milvus_querynode_disk_cache_load_global_duration': 'querynode.disk.cache.load.global_duration', + 'milvus_querynode_disk_cache_load': 'querynode.disk.cache.load', 'milvus_querynode_disk_used_size': 'querynode.disk.used_size', 'milvus_querynode_dml_vchannel_num': 'querynode.dml_vchannel_num', 'milvus_querynode_entity_num': 'querynode.entity.num', 'milvus_querynode_entity_size': 'querynode.entity.size', 'milvus_querynode_execute_bytes_counter': 'querynode.execute_bytes_counter', 'milvus_querynode_flowgraph_num': 'querynode.flowgraph_num', - 'milvus_querynode_forward_delete_latency_sum': 'querynode.forward_delete_latency.sum', - 'milvus_querynode_forward_delete_latency_count': 'querynode.forward_delete_latency.count', - 'milvus_querynode_forward_delete_latency_bucket': 'querynode.forward_delete_latency.bucket', - 'milvus_querynode_load_index_latency_sum': 'querynode.load.index_latency.sum', - 'milvus_querynode_load_index_latency_count': 'querynode.load.index_latency.count', - 'milvus_querynode_load_index_latency_bucket': 'querynode.load.index_latency.bucket', + 'milvus_querynode_forward_delete_latency': 'querynode.forward_delete_latency', + 'milvus_querynode_load_index_latency': 'querynode.load.index_latency', 'milvus_querynode_load_segment_concurrency': 'querynode.load.segment.concurrency', - 'milvus_querynode_load_segment_latency_sum': 'querynode.load.segment.latency.sum', - 'milvus_querynode_load_segment_latency_count': 'querynode.load.segment.latency.count', - 'milvus_querynode_load_segment_latency_bucket': 'querynode.load.segment.latency.bucket', + 'milvus_querynode_load_segment_latency': 'querynode.load.segment.latency', 'milvus_querynode_msg_dispatcher_tt_lag_ms': 'querynode.msg_dispatcher_tt_lag_ms', 'milvus_querynode_partition_num': 'querynode.partition_num', - 'milvus_querynode_process_insert_or_delete_latency_sum': 'querynode.process_insert_or_delete_latency.sum', - 'milvus_querynode_process_insert_or_delete_latency_count': 'querynode.process_insert_or_delete_latency.count', - 'milvus_querynode_process_insert_or_delete_latency_bucket': 'querynode.process_insert_or_delete_latency.bucket', + 'milvus_querynode_process_insert_or_delete_latency': 'querynode.process_insert_or_delete_latency', 'milvus_querynode_read_task_concurrency': 'querynode.read_task.concurrency', 'milvus_querynode_read_task_ready_len': 'querynode.read_task.ready_len', 'milvus_querynode_read_task_unsolved_len': 'querynode.read_task.unsolved_len', - 'milvus_querynode_search_group_nq_sum': 'querynode.search.group.nq.sum', - 'milvus_querynode_search_group_nq_count': 'querynode.search.group.nq.count', - 'milvus_querynode_search_group_nq_bucket': 'querynode.search.group.nq.bucket', - 'milvus_querynode_search_group_size_sum': 'querynode.search.group.size.sum', - 'milvus_querynode_search_group_size_count': 'querynode.search.group.size.count', - 'milvus_querynode_search_group_size_bucket': 'querynode.search.group.size.bucket', - 'milvus_querynode_search_group_topk_sum': 'querynode.search.group.topk.sum', - 'milvus_querynode_search_group_topk_count': 'querynode.search.group.topk.count', - 'milvus_querynode_search_group_topk_bucket': 'querynode.search.group.topk.bucket', - 'milvus_querynode_search_nq_sum': 'querynode.search.nq.sum', - 'milvus_querynode_search_nq_count': 'querynode.search.nq.count', - 'milvus_querynode_search_nq_bucket': 'querynode.search.nq.bucket', - 'milvus_querynode_search_topk_sum': 'querynode.search.topk.sum', - 'milvus_querynode_search_topk_count': 'querynode.search.topk.count', - 'milvus_querynode_search_topk_bucket': 'querynode.search.topk.bucket', + 'milvus_querynode_search_group_nq': 'querynode.search.group.nq', + 'milvus_querynode_search_group_size': 'querynode.search.group.size', + 'milvus_querynode_search_group_topk': 'querynode.search.group.topk', + 'milvus_querynode_search_nq': 'querynode.search.nq', + 'milvus_querynode_search_topk': 'querynode.search.topk', 'milvus_querynode_segment_access_duration': 'querynode.segment.access.duration', - 'milvus_querynode_segment_access_global_duration_sum': 'querynode.segment.access.global_duration.sum', - 'milvus_querynode_segment_access_global_duration_count': 'querynode.segment.access.global_duration.count', - 'milvus_querynode_segment_access_global_duration_bucket': 'querynode.segment.access.global_duration.bucket', - 'milvus_querynode_segment_access_total': 'querynode.segment.access.total', + 'milvus_querynode_segment_access_global_duration': 'querynode.segment.access.global_duration', + 'milvus_querynode_segment_access': 'querynode.segment.access', 'milvus_querynode_segment_access_wait_cache_duration': 'querynode.segment.access.wait_cache.duration', - 'milvus_querynode_segment_access_wait_cache_global_duration_sum': 'querynode.segment.access.wait_cache.global_duration.sum', - 'milvus_querynode_segment_access_wait_cache_global_duration_count': 'querynode.segment.access.wait_cache.global_duration.count', - 'milvus_querynode_segment_access_wait_cache_global_duration_bucket': 'querynode.segment.access.wait_cache.global_duration.bucket', - 'milvus_querynode_segment_access_wait_cache_total': 'querynode.segment.access.wait_cache.total', - 'milvus_querynode_segment_latency_per_vector_sum': 'querynode.segment.latency_per_vector.sum', - 'milvus_querynode_segment_latency_per_vector_count': 'querynode.segment.latency_per_vector.count', - 'milvus_querynode_segment_latency_per_vector_bucket': 'querynode.segment.latency_per_vector.bucket', + 'milvus_querynode_segment_access_wait_cache_global_duration': 'querynode.segment.access.wait_cache.global_duration', + 'milvus_querynode_segment_access_wait_cache': 'querynode.segment.access.wait_cache', + 'milvus_querynode_segment_latency_per_vector': 'querynode.segment.latency_per_vector', 'milvus_querynode_segment_num': 'querynode.segment.num', - 'milvus_querynode_sq_core_latency_sum': 'querynode.sq.core_latency.sum', - 'milvus_querynode_sq_core_latency_count': 'querynode.sq.core_latency.count', - 'milvus_querynode_sq_core_latency_bucket': 'querynode.sq.core_latency.bucket', - 'milvus_querynode_sq_queue_latency_sum': 'querynode.sq.queue.latency.sum', - 'milvus_querynode_sq_queue_latency_count': 'querynode.sq.queue.latency.count', - 'milvus_querynode_sq_queue_latency_bucket': 'querynode.sq.queue.latency.bucket', - 'milvus_querynode_sq_queue_user_latency_sum': 'querynode.sq.queue.user_latency.sum', - 'milvus_querynode_sq_queue_user_latency_count': 'querynode.sq.queue.user_latency.count', - 'milvus_querynode_sq_queue_user_latency_bucket': 'querynode.sq.queue.user_latency.bucket', - 'milvus_querynode_sq_reduce_latency_sum': 'querynode.sq.reduce_latency.sum', - 'milvus_querynode_sq_reduce_latency_count': 'querynode.sq.reduce_latency.count', - 'milvus_querynode_sq_reduce_latency_bucket': 'querynode.sq.reduce_latency.bucket', - 'milvus_querynode_sq_req_count': 'querynode.sq.req.count', - 'milvus_querynode_sq_req_latency_sum': 'querynode.sq.req.latency.sum', - 'milvus_querynode_sq_req_latency_count': 'querynode.sq.req.latency.count', - 'milvus_querynode_sq_req_latency_bucket': 'querynode.sq.req.latency.bucket', - 'milvus_querynode_sq_segment_latency_sum': 'querynode.sq.segment_latency.sum', - 'milvus_querynode_sq_segment_latency_count': 'querynode.sq.segment_latency.count', - 'milvus_querynode_sq_segment_latency_bucket': 'querynode.sq.segment_latency.bucket', - 'milvus_querynode_sq_wait_tsafe_latency_sum': 'querynode.sq.wait_tsafe_latency.sum', - 'milvus_querynode_sq_wait_tsafe_latency_count': 'querynode.sq.wait_tsafe_latency.count', - 'milvus_querynode_sq_wait_tsafe_latency_bucket': 'querynode.sq.wait_tsafe_latency.bucket', - 'milvus_querynode_wait_processing_msg_count': 'querynode.wait_processing_msg_count', - 'milvus_querynode_watch_dml_channel_latency_sum': 'querynode.watch_dml_channel_latency.sum', - 'milvus_querynode_watch_dml_channel_latency_count': 'querynode.watch_dml_channel_latency.count', - 'milvus_querynode_watch_dml_channel_latency_bucket': 'querynode.watch_dml_channel_latency.bucket', + 'milvus_querynode_sq_core_latency': 'querynode.sq.core_latency', + 'milvus_querynode_sq_queue_latency': 'querynode.sq.queue.latency', + 'milvus_querynode_sq_queue_user_latency': 'querynode.sq.queue.user_latency', + 'milvus_querynode_sq_reduce_latency': 'querynode.sq.reduce_latency', + 'milvus_querynode_sq_req_count': 'querynode.sq.req', + 'milvus_querynode_sq_req_latency': 'querynode.sq.req.latency', + 'milvus_querynode_sq_segment_latency': 'querynode.sq.segment_latency', + 'milvus_querynode_sq_wait_tsafe_latency': 'querynode.sq.wait_tsafe_latency', + 'milvus_querynode_wait_processing_msg_count': 'querynode.wait_processing_msg', + 'milvus_querynode_watch_dml_channel_latency': 'querynode.watch_dml_channel_latency', 'milvus_rootcoord_collection_num': 'rootcoord.collection_num', 'milvus_rootcoord_credential_num': 'rootcoord.credential_num', - 'milvus_rootcoord_ddl_req_count': 'rootcoord.ddl_req.count', - 'milvus_rootcoord_ddl_req_latency_sum': 'rootcoord.ddl_req.latency.sum', - 'milvus_rootcoord_ddl_req_latency_count': 'rootcoord.ddl_req.latency.count', - 'milvus_rootcoord_ddl_req_latency_bucket': 'rootcoord.ddl_req.latency.bucket', - 'milvus_rootcoord_ddl_req_latency_in_queue_sum': 'rootcoord.ddl_req.latency_in_queue.sum', - 'milvus_rootcoord_ddl_req_latency_in_queue_count': 'rootcoord.ddl_req.latency_in_queue.count', - 'milvus_rootcoord_ddl_req_latency_in_queue_bucket': 'rootcoord.ddl_req.latency_in_queue.bucket', + 'milvus_rootcoord_ddl_req_count': 'rootcoord.ddl_req', + 'milvus_rootcoord_ddl_req_latency': 'rootcoord.ddl_req.latency', + 'milvus_rootcoord_ddl_req_latency_in_queue': 'rootcoord.ddl_req.latency_in_queue', 'milvus_rootcoord_dml_channel_num': 'rootcoord.dml_channel_num', 'milvus_rootcoord_entity_num': 'rootcoord.entity_num', 'milvus_rootcoord_force_deny_writing_counter': 'rootcoord.force_deny_writing_counter', - 'milvus_rootcoord_id_alloc_count': 'rootcoord.id_alloc_count', + 'milvus_rootcoord_id_alloc_count': 'rootcoord.id_alloc', 'milvus_rootcoord_indexed_entity_num': 'rootcoord.indexed_entity_num', 'milvus_rootcoord_msgstream_obj_num': 'rootcoord.msgstream_obj_num', 'milvus_rootcoord_num_of_roles': 'rootcoord.num_of_roles', @@ -249,61 +145,33 @@ 'milvus_rootcoord_produce_tt_lag_ms': 'rootcoord.produce_tt_lag_ms', 'milvus_rootcoord_proxy_num': 'rootcoord.proxy_num', 'milvus_rootcoord_qn_mem_high_water_level': 'rootcoord.qn_mem_high_water_level', - 'milvus_rootcoord_sync_timetick_latency_sum': 'rootcoord.sync_timetick_latency.sum', - 'milvus_rootcoord_sync_timetick_latency_count': 'rootcoord.sync_timetick_latency.count', - 'milvus_rootcoord_sync_timetick_latency_bucket': 'rootcoord.sync_timetick_latency.bucket', + 'milvus_rootcoord_sync_timetick_latency': 'rootcoord.sync_timetick_latency', 'milvus_rootcoord_timestamp': 'rootcoord.timestamp', 'milvus_rootcoord_timestamp_saved': 'rootcoord.timestamp_saved', 'milvus_runtime_info': 'runtime_info', 'milvus_storage_kv_size': 'storage.kv_size', - 'milvus_storage_op_count': 'storage.op_count', + 'milvus_storage_op_count': 'storage.op', 'milvus_storage_request_latency': 'storage.request_latency', - 'bf_search_cnt_sum': 'bf_search_cnt.sum', - 'bf_search_cnt_count': 'bf_search_cnt.count', - 'bf_search_cnt_bucket': 'bf_search_cnt.bucket', - 'bitset_ratio_sum': 'bitset_ratio.sum', - 'bitset_ratio_count': 'bitset_ratio.count', - 'bitset_ratio_bucket': 'bitset_ratio.bucket', - 'build_latency_sum': 'build_latency.sum', - 'build_latency_count': 'build_latency.count', - 'build_latency_bucket': 'build_latency.bucket', - 'cache_hit_cnt_sum': 'cache_hit_cnt.sum', - 'cache_hit_cnt_count': 'cache_hit_cnt.count', - 'cache_hit_cnt_bucket': 'cache_hit_cnt.bucket', - 'diskann_range_search_iters_sum': 'diskann.range_search_iters.sum', - 'diskann_range_search_iters_count': 'diskann.range_search_iters.count', - 'diskann_range_search_iters_bucket': 'diskann.range_search_iters.bucket', - 'diskann_search_hops_sum': 'diskann.search_hops.sum', - 'diskann_search_hops_count': 'diskann.search_hops.count', - 'diskann_search_hops_bucket': 'diskann.search_hops.bucket', - 'diskann_bitset_ratio_sum': 'diskann_bitset_ratio.sum', - 'diskann_bitset_ratio_count': 'diskann_bitset_ratio.count', - 'diskann_bitset_ratio_bucket': 'diskann_bitset_ratio.bucket', - 'exec_latency_sum': 'exec_latency.sum', - 'exec_latency_count': 'exec_latency.count', - 'exec_latency_bucket': 'exec_latency.bucket', - 'filter_connectivity_ratio_sum': 'filter.connectivity_ratio.sum', - 'filter_connectivity_ratio_count': 'filter.connectivity_ratio.count', - 'filter_connectivity_ratio_bucket': 'filter.connectivity_ratio.bucket', - 'filter_mv_activated_fields_cnt_sum': 'filter.mv.activated_fields_cnt.sum', - 'filter_mv_activated_fields_cnt_count': 'filter.mv.activated_fields_cnt.count', - 'filter_mv_activated_fields_cnt_bucket': 'filter.mv.activated_fields_cnt.bucket', - 'filter_mv_change_base_cnt_sum': 'filter.mv.change_base_cnt.sum', - 'filter_mv_change_base_cnt_count': 'filter.mv.change_base_cnt.count', - 'filter_mv_change_base_cnt_bucket': 'filter.mv.change_base_cnt.bucket', - 'filter_mv_only_cnt_sum': 'filter.mv.only_cnt.sum', - 'filter_mv_only_cnt_count': 'filter.mv.only_cnt.count', - 'filter_mv_only_cnt_bucket': 'filter.mv.only_cnt.bucket', - 'filter_mv_supplement_ep_bool_cnt_sum': 'filter.mv.supplement_ep_bool_cnt.sum', - 'filter_mv_supplement_ep_bool_cnt_count': 'filter.mv.supplement_ep_bool_cnt.count', - 'filter_mv_supplement_ep_bool_cnt_bucket': 'filter.mv.supplement_ep_bool_cnt.bucket', + 'bf_search_cnt': 'bf_search_cnt', + 'bitset_ratio': 'bitset_ratio', + 'build_latency': 'build_latency', + 'cache_hit_cnt': 'cache_hit_cnt', + 'diskann_range_search_iters': 'diskann.range_search_iters', + 'diskann_search_hops': 'diskann.search_hops', + 'diskann_bitset_ratio': 'diskann_bitset_ratio', + 'exec_latency': 'exec_latency', + 'filter_connectivity_ratio': 'filter.connectivity_ratio', + 'filter_mv_activated_fields_cnt': 'filter.mv.activated_fields_cnt', + 'filter_mv_change_base_cnt': 'filter.mv.change_base_cnt', + 'filter_mv_only_cnt': 'filter.mv.only_cnt', + 'filter_mv_supplement_ep_bool_cnt': 'filter.mv.supplement_ep_bool_cnt', 'go_gc_duration_seconds': 'go.gc_duration_seconds', 'go_goroutines': 'go.goroutines', 'go_info': 'go.info', 'go_memstats_alloc_bytes': 'go.memstats.alloc_bytes', - 'go_memstats_alloc_bytes_total': 'go.memstats.alloc_bytes_total', + # 'go_memstats_alloc_bytes_total': 'go.memstats.alloc_bytes_total', 'go_memstats_buck_hash_sys_bytes': 'go.memstats.buck_hash_sys_bytes', - 'go_memstats_frees_total': 'go.memstats.frees_total', + 'go_memstats_frees': 'go.memstats.frees', 'go_memstats_gc_sys_bytes': 'go.memstats.gc_sys_bytes', 'go_memstats_heap_alloc_bytes': 'go.memstats.heap.alloc_bytes', 'go_memstats_heap_idle_bytes': 'go.memstats.heap.idle_bytes', @@ -312,8 +180,8 @@ 'go_memstats_heap_released_bytes': 'go.memstats.heap.released_bytes', 'go_memstats_heap_sys_bytes': 'go.memstats.heap.sys_bytes', 'go_memstats_last_gc_time_seconds': 'go.memstats.last_gc_time_seconds', - 'go_memstats_lookups_total': 'go.memstats.lookups_total', - 'go_memstats_mallocs_total': 'go.memstats.mallocs_total', + 'go_memstats_lookups': 'go.memstats.lookups', + 'go_memstats_mallocs': 'go.memstats.mallocs', 'go_memstats_mcache_inuse_bytes': 'go.memstats.mcache.inuse_bytes', 'go_memstats_mcache_sys_bytes': 'go.memstats.mcache.sys_bytes', 'go_memstats_mspan_inuse_bytes': 'go.memstats.mspan.inuse_bytes', @@ -324,69 +192,33 @@ 'go_memstats_stack_sys_bytes': 'go.memstats.stack.sys_bytes', 'go_memstats_sys_bytes': 'go.memstats.sys_bytes', 'go_threads': 'go.threads', - 'graph_search_cnt_sum': 'graph_search_cnt.sum', - 'graph_search_cnt_count': 'graph_search_cnt.count', - 'graph_search_cnt_bucket': 'graph_search_cnt.bucket', - 'hnsw_bitset_ratio_sum': 'hnsw.bitset_ratio.sum', - 'hnsw_bitset_ratio_count': 'hnsw.bitset_ratio.count', - 'hnsw_bitset_ratio_bucket': 'hnsw.bitset_ratio.bucket', - 'hnsw_search_hops_sum': 'hnsw.search_hops.sum', - 'hnsw_search_hops_count': 'hnsw.search_hops.count', - 'hnsw_search_hops_bucket': 'hnsw.search_hops.bucket', - 'internal_core_search_latency_sum': 'internal.core_search_latency.sum', - 'internal_core_search_latency_count': 'internal.core_search_latency.count', - 'internal_core_search_latency_bucket': 'internal.core_search_latency.bucket', - 'internal_mmap_allocated_space_bytes_sum': 'internal.mmap.allocated_space_bytes.sum', - 'internal_mmap_allocated_space_bytes_count': 'internal.mmap.allocated_space_bytes.count', - 'internal_mmap_allocated_space_bytes_bucket': 'internal.mmap.allocated_space_bytes.bucket', + 'graph_search_cnt': 'graph_search_cnt', + 'hnsw_bitset_ratio': 'hnsw.bitset_ratio', + 'hnsw_search_hops': 'hnsw.search_hops', + 'internal_core_search_latency': 'internal.core_search_latency', + 'internal_mmap_allocated_space_bytes': 'internal.mmap.allocated_space_bytes', 'internal_mmap_in_used_space_bytes': 'internal.mmap.in_used_space_bytes', - 'internal_storage_kv_size_sum': 'internal.storage.kv_size.sum', - 'internal_storage_kv_size_count': 'internal.storage.kv_size.count', - 'internal_storage_kv_size_bucket': 'internal.storage.kv_size.bucket', - 'internal_storage_load_duration_sum': 'internal.storage.load_duration.sum', - 'internal_storage_load_duration_count': 'internal.storage.load_duration.count', - 'internal_storage_load_duration_bucket': 'internal.storage.load_duration.bucket', - 'internal_storage_op_count': 'internal.storage.op_count', - 'internal_storage_request_latency_sum': 'internal.storage.request_latency.sum', - 'internal_storage_request_latency_count': 'internal.storage.request_latency.count', - 'internal_storage_request_latency_bucket': 'internal.storage.request_latency.bucket', - 'io_cnt_sum': 'io_cnt.sum', - 'io_cnt_count': 'io_cnt.count', - 'io_cnt_bucket': 'io_cnt.bucket', - 'ivf_search_cnt_sum': 'ivf_search_cnt.sum', - 'ivf_search_cnt_count': 'ivf_search_cnt.count', - 'ivf_search_cnt_bucket': 'ivf_search_cnt.bucket', - 'load_latency_sum': 'load_latency.sum', - 'load_latency_count': 'load_latency.count', - 'load_latency_bucket': 'load_latency.bucket', - 'process_cpu_seconds_total': 'process.cpu_seconds_total', + 'internal_storage_kv_size': 'internal.storage.kv_size', + 'internal_storage_load_duration': 'internal.storage.load_duration', + 'internal_storage_op_count': 'internal.storage.op', + 'internal_storage_request_latency': 'internal.storage.request_latency', + 'io_cnt': 'io_cnt', + 'ivf_search_cnt': 'ivf_search_cnt', + 'load_latency': 'load_latency', + 'process_cpu_seconds': 'process.cpu_seconds', 'process_max_fds': 'process.max_fds', 'process_open_fds': 'process.open_fds', 'process_resident_memory_bytes': 'process.resident_memory_bytes', 'process_start_time_seconds': 'process.start_time_seconds', 'process_virtual_memory_bytes': 'process.virtual_memory.bytes', 'process_virtual_memory_max_bytes': 'process.virtual_memory.max_bytes', - 'quant_compute_cnt_sum': 'quant.compute_cnt.sum', - 'quant_compute_cnt_count': 'quant.compute_cnt.count', - 'quant_compute_cnt_bucket': 'quant.compute_cnt.bucket', - 'queue_latency_sum': 'queue.latency.sum', - 'queue_latency_count': 'queue.latency.count', - 'queue_latency_bucket': 'queue.latency.bucket', - 'range_search_latency_sum': 'range_search_latency.sum', - 'range_search_latency_count': 'range_search_latency.count', - 'range_search_latency_bucket': 'range_search_latency.bucket', - 'raw_compute_cnt_sum': 'raw_compute_cnt.sum', - 'raw_compute_cnt_count': 'raw_compute_cnt.count', - 'raw_compute_cnt_bucket': 'raw_compute_cnt.bucket', - 're_search_cnt_sum': 're_search_cnt.sum', - 're_search_cnt_count': 're_search_cnt.count', - 're_search_cnt_bucket': 're_search_cnt.bucket', - 'search_latency_sum': 'search.latency.sum', - 'search_latency_count': 'search.latency.count', - 'search_latency_bucket': 'search.latency.bucket', - 'search_topk_sum': 'search.topk.sum', - 'search_topk_count': 'search.topk.count', - 'search_topk_bucket': 'search.topk.bucket', + 'quant_compute_cnt': 'quant.compute_cnt', + 'queue_latency': 'queue.latency', + 'range_search_latency': 'range_search_latency', + 'raw_compute_cnt': 'raw_compute_cnt', + 're_search_cnt': 're_search_cnt', + 'search_latency': 'search.latency', + 'search_topk': 'search.topk' } RENAME_LABELS_MAP = { diff --git a/milvus/metadata.csv b/milvus/metadata.csv index d9ca136437f30..af08773a91419 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -293,7 +293,9 @@ milvus.filter.mv.only_cnt.bucket,count,,,,mv only cnt per request,0,milvus,, milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, -milvus.go.gc_duration_seconds,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.gc_duration_seconds.sum,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.gc_duration_seconds.count,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,, milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, diff --git a/milvus/tests/common.py b/milvus/tests/common.py index e6726ba18a381..af120b86dc964 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -21,142 +21,261 @@ def get_fixture_path(filename): COMPOSE_FILE = os.path.join(HERE, 'docker', 'docker-compose.yaml') -TEST_METRICS = { +STANDALONE_TEST_METRICS = { + 'milvus.go.gc_duration_seconds.quantile': 'gauge', + 'milvus.go.gc_duration_seconds.sum': 'monotonic_count', + 'milvus.go.gc_duration_seconds.count': 'monotonic_count', + 'milvus.go.goroutines': 'gauge', + 'milvus.go.info': 'gauge', + 'milvus.go.memstats.alloc_bytes': 'gauge', + # 'milvus.go.memstats.alloc_bytes.count': 'monotonic_count', + 'milvus.go.memstats.buck_hash_sys_bytes': 'gauge', + 'milvus.go.memstats.frees.count': 'monotonic_count', + 'milvus.go.memstats.gc_sys_bytes': 'gauge', + 'milvus.go.memstats.heap.alloc_bytes': 'gauge', + 'milvus.go.memstats.heap.idle_bytes': 'gauge', + 'milvus.go.memstats.heap.inuse_bytes': 'gauge', + 'milvus.go.memstats.heap.objects': 'gauge', + 'milvus.go.memstats.heap.released_bytes': 'gauge', + 'milvus.go.memstats.heap.sys_bytes': 'gauge', + 'milvus.go.memstats.last_gc_time_seconds': 'gauge', + 'milvus.go.memstats.lookups.count': 'monotonic_count', + 'milvus.go.memstats.mallocs.count': 'monotonic_count', + 'milvus.go.memstats.mcache.inuse_bytes': 'gauge', + 'milvus.go.memstats.mcache.sys_bytes': 'gauge', + 'milvus.go.memstats.mspan.inuse_bytes': 'gauge', + 'milvus.go.memstats.mspan.sys_bytes': 'gauge', + 'milvus.go.memstats.next_gc_bytes': 'gauge', + 'milvus.go.memstats.other_sys_bytes': 'gauge', + 'milvus.go.memstats.stack.inuse_bytes': 'gauge', + 'milvus.go.memstats.stack.sys_bytes': 'gauge', + 'milvus.go.memstats.sys_bytes': 'gauge', + 'milvus.go.threads': 'gauge', 'milvus.build_info': 'gauge', 'milvus.cgo.active_future_total': 'gauge', - 'milvus.cgo.cgo_duration_seconds': 'gauge', - 'milvus.cgo.cgo_queue_duration_seconds': 'gauge', + 'milvus.cgo.cgo_duration_seconds.sum': 'monotonic_count', + 'milvus.cgo.cgo_duration_seconds.count': 'monotonic_count', + 'milvus.cgo.cgo_duration_seconds.bucket': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.sum': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.count': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.bucket': 'monotonic_count', 'milvus.cgo.running_cgo_call_total': 'gauge', - 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', 'milvus.datacoord.collection_num': 'gauge', 'milvus.datacoord.consume_datanode_tt_lag_ms': 'gauge', 'milvus.datacoord.datanode_num': 'gauge', 'milvus.datacoord.import_tasks': 'gauge', 'milvus.datacoord.index.node_num': 'gauge', - 'milvus.datacoord.index.req_count': 'count', - 'milvus.datacoord.index.task_count': 'gauge', + 'milvus.datacoord.index.req.count': 'monotonic_count', 'milvus.datacoord.segment_num': 'gauge', - 'milvus.datacoord.stored.binlog_size': 'gauge', - 'milvus.datacoord.stored.index_files_size': 'gauge', - 'milvus.datacoord.stored.rows_num': 'gauge', - 'milvus.datacoord.task_execute_max_latency': 'gauge', - 'milvus.datacoord.watched_dml_chanel_num': 'gauge', - 'milvus.datanode.autoflush_buffer_op_count': 'count', - 'milvus.datanode.consume.bytes_count': 'count', - 'milvus.datanode.consume.msg_count': 'count', - 'milvus.datanode.consume.tt_lag_ms': 'gauge', - 'milvus.datanode.encode_buffer_latency': 'gauge', + 'milvus.datacoord.task_execute_max_latency.sum': 'monotonic_count', + 'milvus.datacoord.task_execute_max_latency.count': 'monotonic_count', + 'milvus.datacoord.task_execute_max_latency.bucket': 'monotonic_count', + 'milvus.datanode.autoflush_buffer_op.count': 'monotonic_count', + 'milvus.datanode.consume.bytes.count': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.sum': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.count': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.bucket': 'monotonic_count', 'milvus.datanode.flowgraph_num': 'gauge', - 'milvus.datanode.flush.buffer_op_count': 'count', - 'milvus.datanode.flush.req_count': 'count', - 'milvus.datanode.flushed_data.rows': 'count', - 'milvus.datanode.flushed_data.size': 'count', - 'milvus.datanode.msg.dispatcher_tt_lag_ms': 'gauge', - 'milvus.datanode.msg.rows_count': 'count', - 'milvus.datanode.save_latency': 'gauge', - 'milvus.flushed_segment_file_num': 'gauge', - 'milvus.indexnode.build_index_latency': 'gauge', - 'milvus.indexnode.encode_index_latency': 'gauge', - 'milvus.indexnode.index.task_count': 'count', - 'milvus.indexnode.index.task_latency_in_queue': 'gauge', - 'milvus.indexnode.knowhere_build_index_latency': 'gauge', - 'milvus.indexnode.save_index_latency': 'gauge', - 'milvus.meta.kv_size': 'gauge', - 'milvus.meta.op_count': 'count', - 'milvus.meta.request_latency': 'gauge', + 'milvus.datanode.flush.buffer_op.count': 'monotonic_count', + 'milvus.datanode.flush.req.count': 'monotonic_count', + 'milvus.datanode.flushed_data.rows.count': 'monotonic_count', + 'milvus.datanode.flushed_data.size.count': 'monotonic_count', + 'milvus.datanode.msg.rows.count': 'monotonic_count', + 'milvus.datanode.save_latency.sum': 'monotonic_count', + 'milvus.datanode.save_latency.count': 'monotonic_count', + 'milvus.datanode.save_latency.bucket': 'monotonic_count', + 'milvus.flushed_segment_file_num.sum': 'monotonic_count', + 'milvus.flushed_segment_file_num.count': 'monotonic_count', + 'milvus.flushed_segment_file_num.bucket': 'monotonic_count', + 'milvus.indexnode.build_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.build_index_latency.count': 'monotonic_count', + 'milvus.indexnode.build_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.count': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.index.task.count': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.count': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.sum': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.bucket': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.count': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.save_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.save_index_latency.count': 'monotonic_count', + 'milvus.indexnode.save_index_latency.bucket': 'monotonic_count', + 'milvus.meta.kv_size.sum': 'monotonic_count', + 'milvus.meta.kv_size.count': 'monotonic_count', + 'milvus.meta.kv_size.bucket': 'monotonic_count', + 'milvus.meta.op.count': 'monotonic_count', + 'milvus.meta.request_latency.sum': 'monotonic_count', + 'milvus.meta.request_latency.count': 'monotonic_count', + 'milvus.meta.request_latency.bucket': 'monotonic_count', 'milvus.msg_queue_consumer_num': 'gauge', - 'milvus.msgstream.op_count': 'count', - 'milvus.msgstream.request_latency': 'gauge', + 'milvus.msgstream.op.count': 'monotonic_count', + 'milvus.msgstream.request_latency.sum': 'monotonic_count', + 'milvus.msgstream.request_latency.count': 'monotonic_count', + 'milvus.msgstream.request_latency.bucket': 'monotonic_count', 'milvus.num_node': 'gauge', - 'milvus.proxy.apply.pk_latency': 'gauge', - 'milvus.proxy.apply.timestamp_latency': 'gauge', - 'milvus.proxy.assign_segmentID_latency': 'gauge', - 'milvus.proxy.cache.hit_count': 'count', - 'milvus.proxy.cache.update_latency': 'gauge', - 'milvus.proxy.delete_vectors_count': 'count', + 'milvus.proxy.apply.pk_latency.sum': 'monotonic_count', + 'milvus.proxy.apply.pk_latency.count': 'monotonic_count', + 'milvus.proxy.apply.pk_latency.bucket': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.sum': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.count': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.bucket': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.sum': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.count': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.bucket': 'monotonic_count', + 'milvus.proxy.cache.hit.count': 'monotonic_count', + 'milvus.proxy.cache.update_latency.sum': 'monotonic_count', + 'milvus.proxy.cache.update_latency.count': 'monotonic_count', + 'milvus.proxy.cache.update_latency.bucket': 'monotonic_count', + 'milvus.proxy.delete_vectors.count': 'monotonic_count', 'milvus.proxy.msgstream_obj_num': 'gauge', - 'milvus.proxy.mutation_send_latency': 'gauge', - 'milvus.proxy.rate_limit_req_count': 'count', - 'milvus.proxy.report_value': 'count', - 'milvus.proxy.req.count': 'count', - 'milvus.proxy.req.in_queue_latency': 'gauge', - 'milvus.proxy.req.latency': 'gauge', - 'milvus.proxy.send_bytes_count': 'count', - 'milvus.proxy.sq.decode_result_latency': 'gauge', - 'milvus.proxy.sq.reduce_result_latency': 'gauge', - 'milvus.proxy.sq.wait_result_latency': 'gauge', - 'milvus.proxy.sync_segment_request_length': 'gauge', + 'milvus.proxy.mutation_send_latency.sum': 'monotonic_count', + 'milvus.proxy.mutation_send_latency.count': 'monotonic_count', + 'milvus.proxy.mutation_send_latency.bucket': 'monotonic_count', + 'milvus.proxy.rate_limit_req.count': 'monotonic_count', + 'milvus.proxy.report_value.count': 'monotonic_count', + 'milvus.proxy.req.count': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.sum': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.count': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.bucket': 'monotonic_count', + 'milvus.proxy.req.latency.sum': 'monotonic_count', + 'milvus.proxy.req.latency.count': 'monotonic_count', + 'milvus.proxy.req.latency.bucket': 'monotonic_count', + 'milvus.proxy.send_bytes.count': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.sum': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.count': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.bucket': 'monotonic_count', 'milvus.proxy.tt_lag_ms': 'gauge', 'milvus.querycoord.collection_num': 'gauge', - 'milvus.querycoord.current_target_checkpoint_unix_seconds': 'gauge', - 'milvus.querycoord.load.latency': 'gauge', - 'milvus.querycoord.load.req_count': 'count', + 'milvus.querycoord.load.latency.sum': 'monotonic_count', + 'milvus.querycoord.load.latency.count': 'monotonic_count', + 'milvus.querycoord.load.latency.bucket': 'monotonic_count', + 'milvus.querycoord.load.req.count': 'monotonic_count', 'milvus.querycoord.partition_num': 'gauge', 'milvus.querycoord.querynode_num': 'gauge', - 'milvus.querycoord.release.latency': 'gauge', - 'milvus.querycoord.release.req_count': 'count', + 'milvus.querycoord.release.latency.sum': 'monotonic_count', + 'milvus.querycoord.release.latency.count': 'monotonic_count', + 'milvus.querycoord.release.latency.bucket': 'monotonic_count', + 'milvus.querycoord.release.req.count': 'monotonic_count', 'milvus.querycoord_task_num': 'gauge', - 'milvus.querynode.apply_bf_latency': 'gauge', + 'milvus.querynode.apply_bf_latency.sum': 'monotonic_count', + 'milvus.querynode.apply_bf_latency.count': 'monotonic_count', + 'milvus.querynode.apply_bf_latency.bucket': 'monotonic_count', 'milvus.querynode.collection_num': 'gauge', - 'milvus.querynode.consume.bytes_counter': 'count', - 'milvus.querynode.consume.msg_count': 'count', - 'milvus.querynode.consume.tt_lag_ms': 'gauge', - 'milvus.querynode.disk.cache.evict.bytes': 'count', - 'milvus.querynode.disk.cache.evict.duration': 'count', - 'milvus.querynode.disk.cache.evict.global_duration': 'gauge', - 'milvus.querynode.disk.cache.evict.total': 'count', - 'milvus.querynode.disk.cache.load.bytes': 'count', - 'milvus.querynode.disk.cache.load.duration': 'count', - 'milvus.querynode.disk.cache.load.global_duration': 'gauge', - 'milvus.querynode.disk.cache.load.total': 'count', + 'milvus.querynode.consume.bytes_counter.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.bytes.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.sum': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.bytes.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.sum': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.disk.cache.load.count': 'monotonic_count', 'milvus.querynode.disk.used_size': 'gauge', 'milvus.querynode.dml_vchannel_num': 'gauge', - 'milvus.querynode.entity.num': 'gauge', - 'milvus.querynode.entity.size': 'gauge', - 'milvus.querynode.execute_bytes_counter': 'count', + 'milvus.querynode.execute_bytes_counter.count': 'monotonic_count', 'milvus.querynode.flowgraph_num': 'gauge', - 'milvus.querynode.forward_delete_latency': 'gauge', - 'milvus.querynode.load.index_latency': 'gauge', + 'milvus.querynode.forward_delete_latency.sum': 'monotonic_count', + 'milvus.querynode.forward_delete_latency.count': 'monotonic_count', + 'milvus.querynode.forward_delete_latency.bucket': 'monotonic_count', + 'milvus.querynode.load.index_latency.sum': 'monotonic_count', + 'milvus.querynode.load.index_latency.count': 'monotonic_count', + 'milvus.querynode.load.index_latency.bucket': 'monotonic_count', 'milvus.querynode.load.segment.concurrency': 'gauge', - 'milvus.querynode.load.segment.latency': 'gauge', - 'milvus.querynode.msg_dispatcher_tt_lag_ms': 'gauge', + 'milvus.querynode.load.segment.latency.sum': 'monotonic_count', + 'milvus.querynode.load.segment.latency.count': 'monotonic_count', + 'milvus.querynode.load.segment.latency.bucket': 'monotonic_count', 'milvus.querynode.partition_num': 'gauge', - 'milvus.querynode.process_insert_or_delete_latency': 'gauge', + 'milvus.querynode.process_insert_or_delete_latency.sum': 'monotonic_count', + 'milvus.querynode.process_insert_or_delete_latency.count': 'monotonic_count', + 'milvus.querynode.process_insert_or_delete_latency.bucket': 'monotonic_count', 'milvus.querynode.read_task.concurrency': 'gauge', 'milvus.querynode.read_task.ready_len': 'gauge', 'milvus.querynode.read_task.unsolved_len': 'gauge', - 'milvus.querynode.search.group.nq': 'gauge', - 'milvus.querynode.search.group.size': 'gauge', - 'milvus.querynode.search.group.topk': 'gauge', - 'milvus.querynode.search.nq': 'gauge', - 'milvus.querynode.search.topk': 'gauge', - 'milvus.querynode.segment.access.duration': 'count', - 'milvus.querynode.segment.access.global_duration': 'gauge', - 'milvus.querynode.segment.access.total': 'count', - 'milvus.querynode.segment.access.wait_cache.duration': 'count', - 'milvus.querynode.segment.access.wait_cache.global_duration': 'gauge', - 'milvus.querynode.segment.access.wait_cache.total': 'count', - 'milvus.querynode.segment.latency_per_vector': 'gauge', + 'milvus.querynode.search.group.nq.sum': 'monotonic_count', + 'milvus.querynode.search.group.nq.count': 'monotonic_count', + 'milvus.querynode.search.group.nq.bucket': 'monotonic_count', + 'milvus.querynode.search.group.size.sum': 'monotonic_count', + 'milvus.querynode.search.group.size.count': 'monotonic_count', + 'milvus.querynode.search.group.size.bucket': 'monotonic_count', + 'milvus.querynode.search.group.topk.sum': 'monotonic_count', + 'milvus.querynode.search.group.topk.count': 'monotonic_count', + 'milvus.querynode.search.group.topk.bucket': 'monotonic_count', + 'milvus.querynode.search.nq.sum': 'monotonic_count', + 'milvus.querynode.search.nq.count': 'monotonic_count', + 'milvus.querynode.search.nq.bucket': 'monotonic_count', + 'milvus.querynode.search.topk.sum': 'monotonic_count', + 'milvus.querynode.search.topk.count': 'monotonic_count', + 'milvus.querynode.search.topk.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.sum': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.sum': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.count': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.sum': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.count': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.bucket': 'monotonic_count', 'milvus.querynode.segment.num': 'gauge', - 'milvus.querynode.sq.core_latency': 'gauge', - 'milvus.querynode.sq.queue.latency': 'gauge', - 'milvus.querynode.sq.queue.user_latency': 'gauge', - 'milvus.querynode.sq.reduce_latency': 'gauge', - 'milvus.querynode.sq.req.count': 'count', - 'milvus.querynode.sq.req.latency': 'gauge', - 'milvus.querynode.sq.segment_latency': 'gauge', - 'milvus.querynode.sq.wait_tsafe_latency': 'gauge', - 'milvus.querynode.wait_processing_msg_count': 'gauge', - 'milvus.querynode.watch_dml_channel_latency': 'gauge', + 'milvus.querynode.sq.core_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.core_latency.count': 'monotonic_count', + 'milvus.querynode.sq.core_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.sum': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.count': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.count': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.count': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.req.count': 'monotonic_count', + 'milvus.querynode.sq.req.latency.sum': 'monotonic_count', + 'milvus.querynode.sq.req.latency.count': 'monotonic_count', + 'milvus.querynode.sq.req.latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.count': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.count': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.bucket': 'monotonic_count', + 'milvus.querynode.wait_processing_msg': 'gauge', + 'milvus.querynode.watch_dml_channel_latency.sum': 'monotonic_count', + 'milvus.querynode.watch_dml_channel_latency.count': 'monotonic_count', + 'milvus.querynode.watch_dml_channel_latency.bucket': 'monotonic_count', 'milvus.rootcoord.collection_num': 'gauge', 'milvus.rootcoord.credential_num': 'gauge', - 'milvus.rootcoord.ddl_req.count': 'count', - 'milvus.rootcoord.ddl_req.latency': 'gauge', - 'milvus.rootcoord.ddl_req.latency_in_queue': 'gauge', + 'milvus.rootcoord.ddl_req.count': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.sum': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.sum': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.count': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.bucket': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.sum': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.count': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.bucket': 'monotonic_count', 'milvus.rootcoord.dml_channel_num': 'gauge', 'milvus.rootcoord.entity_num': 'gauge', - 'milvus.rootcoord.force_deny_writing_counter': 'count', - 'milvus.rootcoord.id_alloc_count': 'count', + 'milvus.rootcoord.force_deny_writing_counter.count': 'monotonic_count', + 'milvus.rootcoord.id_alloc.count': 'monotonic_count', 'milvus.rootcoord.indexed_entity_num': 'gauge', 'milvus.rootcoord.msgstream_obj_num': 'gauge', 'milvus.rootcoord.num_of_roles': 'gauge', @@ -164,15 +283,137 @@ def get_fixture_path(filename): 'milvus.rootcoord.produce_tt_lag_ms': 'gauge', 'milvus.rootcoord.proxy_num': 'gauge', 'milvus.rootcoord.qn_mem_high_water_level': 'gauge', - 'milvus.rootcoord.sync_timetick_latency': 'gauge', - 'milvus.rootcoord.timestamp': 'gauge', 'milvus.rootcoord.timestamp_saved': 'gauge', 'milvus.runtime_info': 'gauge', + 'milvus.process.cpu_seconds.count': 'monotonic_count', + 'milvus.process.max_fds': 'gauge', + 'milvus.process.open_fds': 'gauge', + 'milvus.process.resident_memory_bytes': 'gauge', + 'milvus.process.start_time_seconds': 'gauge', + 'milvus.process.virtual_memory.bytes': 'gauge', + 'milvus.process.virtual_memory.max_bytes': 'gauge', + 'milvus.bf_search_cnt.sum': 'monotonic_count', + 'milvus.bf_search_cnt.count': 'monotonic_count', + 'milvus.bf_search_cnt.bucket': 'monotonic_count', + 'milvus.bitset_ratio.sum': 'monotonic_count', + 'milvus.bitset_ratio.count': 'monotonic_count', + 'milvus.bitset_ratio.bucket': 'monotonic_count', + 'milvus.build_latency.sum': 'monotonic_count', + 'milvus.build_latency.count': 'monotonic_count', + 'milvus.build_latency.bucket': 'monotonic_count', + 'milvus.cache_hit_cnt.sum': 'monotonic_count', + 'milvus.cache_hit_cnt.count': 'monotonic_count', + 'milvus.cache_hit_cnt.bucket': 'monotonic_count', + 'milvus.diskann_bitset_ratio.sum': 'monotonic_count', + 'milvus.diskann_bitset_ratio.count': 'monotonic_count', + 'milvus.diskann_bitset_ratio.bucket': 'monotonic_count', + 'milvus.diskann.range_search_iters.sum': 'monotonic_count', + 'milvus.diskann.range_search_iters.count': 'monotonic_count', + 'milvus.diskann.range_search_iters.bucket': 'monotonic_count', + 'milvus.diskann.search_hops.sum': 'monotonic_count', + 'milvus.diskann.search_hops.count': 'monotonic_count', + 'milvus.diskann.search_hops.bucket': 'monotonic_count', + 'milvus.exec_latency.sum': 'monotonic_count', + 'milvus.exec_latency.count': 'monotonic_count', + 'milvus.exec_latency.bucket': 'monotonic_count', + 'milvus.filter.connectivity_ratio.sum': 'monotonic_count', + 'milvus.filter.connectivity_ratio.count': 'monotonic_count', + 'milvus.filter.connectivity_ratio.bucket': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.count': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.count': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.only_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.only_cnt.count': 'monotonic_count', + 'milvus.filter.mv.only_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.count': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.bucket': 'monotonic_count', + 'milvus.graph_search_cnt.sum': 'monotonic_count', + 'milvus.graph_search_cnt.count': 'monotonic_count', + 'milvus.graph_search_cnt.bucket': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.sum': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.count': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.bucket': 'monotonic_count', + 'milvus.hnsw.search_hops.sum': 'monotonic_count', + 'milvus.hnsw.search_hops.count': 'monotonic_count', + 'milvus.hnsw.search_hops.bucket': 'monotonic_count', + 'milvus.internal.core_search_latency.sum': 'monotonic_count', + 'milvus.internal.core_search_latency.count': 'monotonic_count', + 'milvus.internal.core_search_latency.bucket': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.sum': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.count': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.bucket': 'monotonic_count', + 'milvus.internal.mmap.in_used_space_bytes': 'gauge', + 'milvus.internal.storage.kv_size.sum': 'monotonic_count', + 'milvus.internal.storage.kv_size.count': 'monotonic_count', + 'milvus.internal.storage.kv_size.bucket': 'monotonic_count', + 'milvus.internal.storage.load_duration.sum': 'monotonic_count', + 'milvus.internal.storage.load_duration.count': 'monotonic_count', + 'milvus.internal.storage.load_duration.bucket': 'monotonic_count', + 'milvus.internal.storage.op.count': 'monotonic_count', + 'milvus.internal.storage.request_latency.sum': 'monotonic_count', + 'milvus.internal.storage.request_latency.count': 'monotonic_count', + 'milvus.internal.storage.request_latency.bucket': 'monotonic_count', + 'milvus.io_cnt.sum': 'monotonic_count', + 'milvus.io_cnt.count': 'monotonic_count', + 'milvus.io_cnt.bucket': 'monotonic_count', + 'milvus.ivf_search_cnt.sum': 'monotonic_count', + 'milvus.ivf_search_cnt.count': 'monotonic_count', + 'milvus.ivf_search_cnt.bucket': 'monotonic_count', + 'milvus.load_latency.sum': 'monotonic_count', + 'milvus.load_latency.count': 'monotonic_count', + 'milvus.load_latency.bucket': 'monotonic_count', + 'milvus.quant.compute_cnt.sum': 'monotonic_count', + 'milvus.quant.compute_cnt.count': 'monotonic_count', + 'milvus.quant.compute_cnt.bucket': 'monotonic_count', + 'milvus.queue.latency.sum': 'monotonic_count', + 'milvus.queue.latency.count': 'monotonic_count', + 'milvus.queue.latency.bucket': 'monotonic_count', + 'milvus.range_search_latency.sum': 'monotonic_count', + 'milvus.range_search_latency.count': 'monotonic_count', + 'milvus.range_search_latency.bucket': 'monotonic_count', + 'milvus.raw_compute_cnt.sum': 'monotonic_count', + 'milvus.raw_compute_cnt.count': 'monotonic_count', + 'milvus.raw_compute_cnt.bucket': 'monotonic_count', + 'milvus.re_search_cnt.sum': 'monotonic_count', + 'milvus.re_search_cnt.count': 'monotonic_count', + 'milvus.re_search_cnt.bucket': 'monotonic_count', + 'milvus.search.latency.sum': 'monotonic_count', + 'milvus.search.latency.count': 'monotonic_count', + 'milvus.search.latency.bucket': 'monotonic_count', + 'milvus.search.topk.sum': 'monotonic_count', + 'milvus.search.topk.count': 'monotonic_count', + 'milvus.search.topk.bucket': 'monotonic_count', + } + +OTHER_TEST_METRICS = { + 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', + 'milvus.datacoord.index.task_count': 'gauge', + 'milvus.datacoord.stored.binlog_size': 'gauge', + 'milvus.datacoord.stored.index_files_size': 'gauge', + 'milvus.datacoord.stored.rows_num': 'gauge', + 'milvus.datacoord.watched_dml_chanel_num': 'gauge', + 'milvus.datanode.consume.msg_count': 'count', + 'milvus.datanode.consume.tt_lag_ms': 'gauge', + 'milvus.datanode.msg.dispatcher_tt_lag_ms': 'gauge', + 'milvus.querycoord.current_target_checkpoint_unix_seconds': 'gauge', + 'milvus.querynode.consume.msg_count': 'count', + 'milvus.querynode.consume.tt_lag_ms': 'gauge', + 'milvus.querynode.entity.num': 'gauge', + 'milvus.querynode.entity.size': 'gauge', + 'milvus.querynode.msg_dispatcher_tt_lag_ms': 'gauge', + 'milvus.rootcoord.sync_timetick_latency.bucket': 'count', + 'milvus.rootcoord.sync_timetick_latency.count': 'count', + 'milvus.rootcoord.timestamp': 'gauge', 'milvus.storage.kv_size': 'gauge', 'milvus.storage.op_count': 'count', 'milvus.storage.request_latency': 'gauge', } + RENAMED_LABELS = { - "milvus.build_info": 'milvus_version:v2.3.15', + "milvus.build_info": 'milvus_version:v2.4.15', } diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py index 52fee01ea157b..23af6f5d0c53a 100644 --- a/milvus/tests/test_unit.py +++ b/milvus/tests/test_unit.py @@ -7,7 +7,7 @@ from datadog_checks.dev.utils import get_metadata_metrics from datadog_checks.milvus import MilvusCheck -from .common import TEST_METRICS, RENAMED_LABELS, get_fixture_path +from .common import RENAMED_LABELS, STANDALONE_TEST_METRICS, get_fixture_path def test_check(dd_run_check, aggregator, instance, mock_http_response): @@ -16,7 +16,7 @@ def test_check(dd_run_check, aggregator, instance, mock_http_response): check = MilvusCheck('milvus', {}, [instance]) dd_run_check(check) - for metric, metric_type in TEST_METRICS.items(): + for metric, metric_type in STANDALONE_TEST_METRICS.items(): aggregator.assert_metric(metric, metric_type=aggregator.METRIC_ENUM_MAP[metric_type]) aggregator.assert_metric_has_tag(metric, 'test:tag') @@ -27,12 +27,6 @@ def test_check(dd_run_check, aggregator, instance, mock_http_response): aggregator.assert_metrics_using_metadata(get_metadata_metrics()) -def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance): - check = MilvusCheck('milvus', {}, [instance]) - dd_run_check(check) - aggregator.assert_service_check('milvus.can_connect', MilvusCheck.CRITICAL) - - def test_empty_instance(dd_run_check): with pytest.raises( Exception, From c2c4bf5b4856303654929bc9cbc6fdca68403bd8 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Thu, 12 Dec 2024 11:32:31 +0100 Subject: [PATCH 18/31] Passing unit tests --- .../milvus/config_models/validators.py | 45 ++++++++-- milvus/metadata.csv | 88 +++++++++---------- milvus/tests/common.py | 3 + 3 files changed, 83 insertions(+), 53 deletions(-) diff --git a/milvus/datadog_checks/milvus/config_models/validators.py b/milvus/datadog_checks/milvus/config_models/validators.py index 70150e85e6124..1a91a82658202 100644 --- a/milvus/datadog_checks/milvus/config_models/validators.py +++ b/milvus/datadog_checks/milvus/config_models/validators.py @@ -1,13 +1,40 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from urllib.parse import urlparse -# Here you can include additional config validators or transformers -# -# def initialize_instance(values, **kwargs): -# if 'my_option' not in values and 'my_legacy_option' in values: -# values['my_option'] = values['my_legacy_option'] -# if values.get('my_number') > 10: -# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) -# -# return values + +def initialize_instance(values, **kwargs): + if 'openmetrics_endpoint' in values: + validate_url( + values['openmetrics_endpoint'], + required_path='/metrics', + example='http://localhost:9091/metrics', + config='openmetrics_endpoint', + ) + + return values + + +def validate_url(url, required_path=None, example=None, config=None): + # Validates the endpoint to ensure that the components are present. + # For OpenMetrics: Scheme, netloc and /metrics path + # For API: Scheme, netloc and no path + # Logs out troubleshooting errors and example + url_parsed = urlparse(url) + errors = [] + + if not url_parsed.netloc: + errors.append("couldn't properly parse endpoint") + if not url_parsed.scheme: + errors.append("http or https scheme is missing") + if required_path and url_parsed.path != required_path: + errors.append(f"URL should end with {required_path}") + if not required_path and url_parsed.path: + errors.append("should not contain a path or trailing /") + + if errors: + error_message = ", ".join(errors) + raise ValueError( + f"{config}: {url} is incorrectly configured. Errors detected: {error_message}. Example: {example}" + ) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index af08773a91419..14e3ec6cdeca5 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -14,7 +14,7 @@ milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus t milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, -milvus.datacoord.index.req_count,count,,,,number of building index requests,0,milvus,, +milvus.datacoord.index.req.count,count,,,,number of building index requests,0,milvus,, milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, @@ -24,20 +24,20 @@ milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,latency of tas milvus.datacoord.task_execute_max_latency.count,count,,millisecond,,latency of task execute operation,0,milvus,, milvus.datacoord.task_execute_max_latency.bucket,count,,millisecond,,latency of task execute operation,0,milvus,, milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, -milvus.datanode.autoflush_buffer_op_count,count,,,,count of auto flush buffer operations,0,milvus,, -milvus.datanode.consume.bytes_count,count,,,,,0,milvus,, -milvus.datanode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.datanode.autoflush_buffer_op.count,count,,,,count of auto flush buffer operations,0,milvus,, +milvus.datanode.consume.bytes.count,count,,,,,0,milvus,, +milvus.datanode.consume.msg.count,count,,,,count of consumed msg,0,milvus,, milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,latency of encode buffer data,0,milvus,, milvus.datanode.encode_buffer_latency.count,count,,millisecond,,latency of encode buffer data,0,milvus,, milvus.datanode.encode_buffer_latency.bucket,count,,millisecond,,latency of encode buffer data,0,milvus,, milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.datanode.flush.buffer_op_count,count,,,,count of flush buffer operations,0,milvus,, -milvus.datanode.flush.req_count,count,,,,count of flush request,0,milvus,, -milvus.datanode.flushed_data.rows,count,,,,num of rows flushed to storage,0,milvus,, -milvus.datanode.flushed_data.size,count,,byte,,byte size of data flushed to storage,0,milvus,, +milvus.datanode.flush.buffer_op.count,count,,,,count of flush buffer operations,0,milvus,, +milvus.datanode.flush.req.count,count,,,,count of flush request,0,milvus,, +milvus.datanode.flushed_data.rows.count,count,,,,num of rows flushed to storage,0,milvus,, +milvus.datanode.flushed_data.size.count,count,,byte,,byte size of data flushed to storage,0,milvus,, milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus.datanode.msg.rows_count,count,,,,count of rows consumed from msgStream,0,milvus,, +milvus.datanode.msg.rows.count,count,,,,count of rows consumed from msgStream,0,milvus,, milvus.datanode.save_latency.sum,count,,millisecond,,latency of saving flush data to storage,0,milvus,, milvus.datanode.save_latency.count,count,,millisecond,,latency of saving flush data to storage,0,milvus,, milvus.datanode.save_latency.bucket,count,,millisecond,,latency of saving flush data to storage,0,milvus,, @@ -50,7 +50,7 @@ milvus.indexnode.build_index_latency.bucket,count,,millisecond,,latency of build milvus.indexnode.encode_index_latency.sum,count,,millisecond,,latency of encoding the index file,0,milvus,, milvus.indexnode.encode_index_latency.count,count,,millisecond,,latency of encoding the index file,0,milvus,, milvus.indexnode.encode_index_latency.bucket,count,,millisecond,,latency of encoding the index file,0,milvus,, -milvus.indexnode.index.task_count,count,,,,number of tasks that index node received,0,milvus,, +milvus.indexnode.index.task.count,count,,,,number of tasks that index node received,0,milvus,, milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,latency of index task in queue,0,milvus,, milvus.indexnode.index.task_latency_in_queue.count,count,,millisecond,,latency of index task in queue,0,milvus,, milvus.indexnode.index.task_latency_in_queue.bucket,count,,millisecond,,latency of index task in queue,0,milvus,, @@ -63,12 +63,12 @@ milvus.indexnode.save_index_latency.bucket,count,,millisecond,,latency of saving milvus.meta.kv_size.sum,count,,,,kv size stats,0,milvus,, milvus.meta.kv_size.count,count,,,,kv size stats,0,milvus,, milvus.meta.kv_size.bucket,count,,,,kv size stats,0,milvus,, -milvus.meta.op_count,count,,,,count of meta operation,0,milvus,, +milvus.meta.op.count,count,,,,count of meta operation,0,milvus,, milvus.meta.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, milvus.meta.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, milvus.meta.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, -milvus.msgstream.op_count,count,,,,count of stream message operation,0,milvus,, +milvus.msgstream.op.count,count,,,,count of stream message operation,0,milvus,, milvus.msgstream.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, milvus.msgstream.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, milvus.msgstream.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, @@ -82,17 +82,17 @@ milvus.proxy.apply.timestamp_latency.bucket,count,,millisecond,,latency that pro milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, milvus.proxy.assign_segmentID_latency.count,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, milvus.proxy.assign_segmentID_latency.bucket,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.proxy.cache.hit_count,count,,,,count of cache hits/miss,0,milvus,, +milvus.proxy.cache.hit.count,count,,,,count of cache hits/miss,0,milvus,, milvus.proxy.cache.update_latency.sum,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, milvus.proxy.cache.update_latency.count,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, milvus.proxy.cache.update_latency.bucket,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, -milvus.proxy.delete_vectors_count,count,,,,counter of vectors successfully deleted,0,milvus,, +milvus.proxy.delete_vectors.count,count,,,,counter of vectors successfully deleted,0,milvus,, milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, milvus.proxy.mutation_send_latency.sum,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, milvus.proxy.mutation_send_latency.count,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, milvus.proxy.mutation_send_latency.bucket,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.proxy.rate_limit_req_count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.report_value,count,,,,report value about the request,0,milvus,, +milvus.proxy.rate_limit_req.count,count,,,,count of operation executed,0,milvus,, +milvus.proxy.report_value.count,count,,,,report value about the request,0,milvus,, milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,latency which request waits in the queue,0,milvus,, milvus.proxy.req.in_queue_latency.count,count,,millisecond,,latency which request waits in the queue,0,milvus,, @@ -100,7 +100,7 @@ milvus.proxy.req.in_queue_latency.bucket,count,,millisecond,,latency which reque milvus.proxy.req.latency.sum,count,,millisecond,,latency of each request,0,milvus,, milvus.proxy.req.latency.count,count,,millisecond,,latency of each request,0,milvus,, milvus.proxy.req.latency.bucket,count,,millisecond,,latency of each request,0,milvus,, -milvus.proxy.send_bytes_count,count,,,,count of bytes sent back to sdk,0,milvus,, +milvus.proxy.send_bytes.count,count,,,,count of bytes sent back to sdk,0,milvus,, milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, milvus.proxy.sq.decode_result_latency.count,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, milvus.proxy.sq.decode_result_latency.bucket,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, @@ -119,38 +119,38 @@ milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target milvus.querycoord.load.latency.sum,count,,millisecond,,latency of load the entire collection,0,milvus,, milvus.querycoord.load.latency.count,count,,millisecond,,latency of load the entire collection,0,milvus,, milvus.querycoord.load.latency.bucket,count,,millisecond,,latency of load the entire collection,0,milvus,, -milvus.querycoord.load.req_count,count,,,,count of load request,0,milvus,, +milvus.querycoord.load.req.count,count,,,,count of load request,0,milvus,, milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, milvus.querycoord.release.latency.sum,count,,millisecond,,latency of release request,0,milvus,, milvus.querycoord.release.latency.count,count,,millisecond,,latency of release request,0,milvus,, milvus.querycoord.release.latency.bucket,count,,millisecond,,latency of release request,0,milvus,, -milvus.querycoord.release.req_count,count,,,,count of release request,0,milvus,, +milvus.querycoord.release.req.count,count,,,,count of release request,0,milvus,, milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, milvus.querynode.apply_bf_latency.sum,count,,millisecond,,apply bf cost in ms,0,milvus,, milvus.querynode.apply_bf_latency.count,count,,millisecond,,apply bf cost in ms,0,milvus,, milvus.querynode.apply_bf_latency.bucket,count,,millisecond,,apply bf cost in ms,0,milvus,, milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, -milvus.querynode.consume.bytes_counter,count,,,,,0,milvus,, -milvus.querynode.consume.msg_count,count,,,,count of consumed msg,0,milvus,, +milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,, +milvus.querynode.consume.msg.count,count,,,,count of consumed msg,0,milvus,, milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.querynode.disk.cache.evict.bytes,count,,byte,,number of bytes evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.duration,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,number of bytes evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.duration.count,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.global_duration.count,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, milvus.querynode.disk.cache.evict.global_duration.bucket,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.total,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.bytes,count,,byte,,number of bytes loaded from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.duration,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, +milvus.querynode.disk.cache.evict.count,count,,,,number of segments evicted from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.bytes.count,count,,byte,,number of bytes loaded from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.duration.count,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, milvus.querynode.disk.cache.load.global_duration.count,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, milvus.querynode.disk.cache.load.global_duration.bucket,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.total,count,,,,number of segments loaded from disk cache,0,milvus,, +milvus.querynode.disk.cache.load.count,count,,,,number of segments loaded from disk cache,0,milvus,, milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, -milvus.querynode.execute_bytes_counter,count,,,,,0,milvus,, +milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,, milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, milvus.querynode.forward_delete_latency.sum,count,,millisecond,,forward delete cost in ms,0,milvus,, milvus.querynode.forward_delete_latency.count,count,,millisecond,,forward delete cost in ms,0,milvus,, @@ -185,16 +185,16 @@ milvus.querynode.search.nq.bucket,count,,,,the number of queries of each search milvus.querynode.search.topk.sum,count,,,,the top of each search task,0,milvus,, milvus.querynode.search.topk.count,count,,,,the top of each search task,0,milvus,, milvus.querynode.search.topk.bucket,count,,,,the top of each search task,0,milvus,, -milvus.querynode.segment.access.duration,count,,millisecond,,total time cost of accessing segments,0,milvus,, +milvus.querynode.segment.access.duration.count,count,,millisecond,,total time cost of accessing segments,0,milvus,, milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,global time cost of accessing segments,0,milvus,, milvus.querynode.segment.access.global_duration.count,count,,millisecond,,global time cost of accessing segments,0,milvus,, milvus.querynode.segment.access.global_duration.bucket,count,,millisecond,,global time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.total,count,,,,number of segments accessed,0,milvus,, -milvus.querynode.segment.access.wait_cache.duration,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, +milvus.querynode.segment.access.count,count,,,,number of segments accessed,0,milvus,, +milvus.querynode.segment.access.wait_cache.duration.count,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, milvus.querynode.segment.access.wait_cache.global_duration.count,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.total,count,,,,number of segments waiting for loading access,0,milvus,, +milvus.querynode.segment.access.wait_cache.count,count,,,,number of segments waiting for loading access,0,milvus,, milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,one vector's search latency per segment,0,milvus,, milvus.querynode.segment.latency_per_vector.count,count,,millisecond,,one vector's search latency per segment,0,milvus,, milvus.querynode.segment.latency_per_vector.bucket,count,,millisecond,,one vector's search latency per segment,0,milvus,, @@ -221,7 +221,7 @@ milvus.querynode.sq.segment_latency.bucket,count,,millisecond,,latency of search milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, milvus.querynode.sq.wait_tsafe_latency.count,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, milvus.querynode.sq.wait_tsafe_latency.bucket,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, -milvus.querynode.wait_processing_msg_count,gauge,,,,count of wait processing msg,0,milvus,, +milvus.querynode.wait_processing_msg,gauge,,,.count,count of wait processing msg,0,milvus,, milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,latency of watch dml channel,0,milvus,, milvus.querynode.watch_dml_channel_latency.count,count,,millisecond,,latency of watch dml channel,0,milvus,, milvus.querynode.watch_dml_channel_latency.bucket,count,,millisecond,,latency of watch dml channel,0,milvus,, @@ -236,8 +236,8 @@ milvus.rootcoord.ddl_req.latency_in_queue.count,count,,millisecond,,latency of e milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, -milvus.rootcoord.force_deny_writing_counter,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, -milvus.rootcoord.id_alloc_count,count,,,,count of ID allocated,0,milvus,, +milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, +milvus.rootcoord.id_alloc.count,count,,,,count of ID allocated,0,milvus,, milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, @@ -252,7 +252,7 @@ milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvu milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, -milvus.storage.op_count,count,,,,count of persistent data operation,0,milvus,, +milvus.storage.op.count,count,,,,count of persistent data operation,0,milvus,, milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, milvus.bf_search_cnt.sum,count,,,,number of bf search per request,0,milvus,, milvus.bf_search_cnt.count,count,,,,number of bf search per request,0,milvus,, @@ -294,14 +294,14 @@ milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitse milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go.gc_duration_seconds.sum,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go.gc_duration_seconds.count,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, +milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,, milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, -milvus.go.memstats.alloc_bytes_total,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, +milvus.go.memstats.alloc_bytes.count,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, milvus.go.memstats.buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,, -milvus.go.memstats.frees_total,count,,,,Total number of frees.,0,milvus,, +milvus.go.memstats.frees.count,count,,,,Total number of frees.,0,milvus,, milvus.go.memstats.gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,, milvus.go.memstats.heap.alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,, milvus.go.memstats.heap.idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,, @@ -310,8 +310,8 @@ milvus.go.memstats.heap.objects,gauge,,,,Number of allocated objects.,0,milvus,, milvus.go.memstats.heap.released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,, milvus.go.memstats.heap.sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,, milvus.go.memstats.last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,, -milvus.go.memstats.lookups_total,count,,,,Total number of pointer lookups.,0,milvus,, -milvus.go.memstats.mallocs_total,count,,,,Total number of mallocs.,0,milvus,, +milvus.go.memstats.lookups.count,count,,,,Total number of pointer lookups.,0,milvus,, +milvus.go.memstats.mallocs.count,count,,,,Total number of mallocs.,0,milvus,, milvus.go.memstats.mcache.inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,, milvus.go.memstats.mcache.sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,, milvus.go.memstats.mspan.inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,, @@ -344,7 +344,7 @@ milvus.internal.storage.kv_size.bucket,count,,,,[cpp]kv size stats,0,milvus,, milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segment,0,milvus,, milvus.internal.storage.load_duration.count,count,,,,[cpp]durations of load segment,0,milvus,, milvus.internal.storage.load_duration.bucket,count,,,,[cpp]durations of load segment,0,milvus,, -milvus.internal.storage.op_count,count,,,,[cpp]count of persistent data operation,0,milvus,, +milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,, milvus.internal.storage.request_latency.sum,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, @@ -357,7 +357,7 @@ milvus.ivf_search_cnt.bucket,count,,,,number of ivf search per request,0,milvus, milvus.load_latency.sum,count,,,,index load latency (ms),0,milvus,, milvus.load_latency.count,count,,,,index load latency (ms),0,milvus,, milvus.load_latency.bucket,count,,,,index load latency (ms),0,milvus,, -milvus.process.cpu_seconds_total,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, +milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,, diff --git a/milvus/tests/common.py b/milvus/tests/common.py index af120b86dc964..6a58d719f4e3b 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -266,6 +266,8 @@ def get_fixture_path(filename): 'milvus.rootcoord.credential_num': 'gauge', 'milvus.rootcoord.ddl_req.count': 'monotonic_count', 'milvus.rootcoord.sync_timetick_latency.sum': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.bucket': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.count': 'monotonic_count', 'milvus.rootcoord.ddl_req.latency.sum': 'monotonic_count', 'milvus.rootcoord.ddl_req.latency.count': 'monotonic_count', 'milvus.rootcoord.ddl_req.latency.bucket': 'monotonic_count', @@ -283,6 +285,7 @@ def get_fixture_path(filename): 'milvus.rootcoord.produce_tt_lag_ms': 'gauge', 'milvus.rootcoord.proxy_num': 'gauge', 'milvus.rootcoord.qn_mem_high_water_level': 'gauge', + 'milvus.rootcoord.timestamp': 'gauge', 'milvus.rootcoord.timestamp_saved': 'gauge', 'milvus.runtime_info': 'gauge', 'milvus.process.cpu_seconds.count': 'monotonic_count', From b95c1239f52082fac13ed921e13bf43e71afddd9 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Fri, 13 Dec 2024 18:09:04 +0100 Subject: [PATCH 19/31] Add e2e env and test --- milvus/hatch.toml | 2 +- milvus/images/IMAGES_README.md | 41 ------ milvus/tests/common.py | 2 +- milvus/tests/compose/Dockerfile | 13 ++ milvus/tests/compose/docker-compose.yaml | 41 ++++++ milvus/tests/compose/embedEtcd.yaml | 5 + milvus/tests/compose/requirements.txt | 2 + milvus/tests/compose/script.py | 155 +++++++++++++++++++++++ milvus/tests/compose/user.yaml | 1 + milvus/tests/conftest.py | 4 +- milvus/tests/docker/Caddyfile | 4 - milvus/tests/docker/docker-compose.yaml | 11 -- milvus/tests/test_e2e.py | 20 +++ 13 files changed, 242 insertions(+), 59 deletions(-) delete mode 100644 milvus/images/IMAGES_README.md create mode 100644 milvus/tests/compose/Dockerfile create mode 100644 milvus/tests/compose/docker-compose.yaml create mode 100644 milvus/tests/compose/embedEtcd.yaml create mode 100644 milvus/tests/compose/requirements.txt create mode 100644 milvus/tests/compose/script.py create mode 100644 milvus/tests/compose/user.yaml delete mode 100644 milvus/tests/docker/Caddyfile delete mode 100644 milvus/tests/docker/docker-compose.yaml create mode 100644 milvus/tests/test_e2e.py diff --git a/milvus/hatch.toml b/milvus/hatch.toml index 001e43ce25414..c85c5f07a7df2 100644 --- a/milvus/hatch.toml +++ b/milvus/hatch.toml @@ -1,4 +1,4 @@ [env.collectors.datadog-checks] [[envs.default.matrix]] -python = ["3.11"] +python = ["3.12"] diff --git a/milvus/images/IMAGES_README.md b/milvus/images/IMAGES_README.md deleted file mode 100644 index 443f3c45e3385..0000000000000 --- a/milvus/images/IMAGES_README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Marketplace Media Carousel Guidelines - -## Using the media gallery - -Please upload images to use the media gallery. Integrations require a minimum of 3 images. Images should highlight your product, your integration, and a full image of the Datadog integration dashboard. The gallery -can hold a maximum of 8 pieces of media total, and one of these pieces of media -can be a video (guidelines and submission steps below). Images should be -added to your /images directory and referenced in the manifest.json file. - - -## Image and video requirements - -### Images - -``` -File type : .jpg or .png -File size : ~500 KB per image, with a max of 1 MB per image -File dimensions : The image must be between 1440px and 2880px width, with a 16:9 aspect ratio (for example: 1440x810) -File name : Use only letters, numbers, underscores, and hyphens -Color mode : RGB -Color profile : sRGB -Description : 300 characters maximum -``` - -### Video - -To display a video in your media gallery, please send our team the zipped file -or a link to download the video at `marketplace@datadog.com`. In addition, -please upload a thumbnail image for your video as a part of the pull request. -Once approved, we will upload the file to Vimeo and provide you with the -vimeo_id to add to your manifest.json file. Please note that the gallery can -only hold one video. - -``` -File type : MP4 H.264 -File size : Max 1 video; 1 GB maximum size -File dimensions : The aspect ratio must be exactly 16:9, and the resolution must be 1920x1080 or higher -File name : partnerName-appName.mp4 -Run time : Recommendation of 60 seconds or less -Description : 300 characters maximum -``` diff --git a/milvus/tests/common.py b/milvus/tests/common.py index 6a58d719f4e3b..1e84b66583a8a 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -19,7 +19,7 @@ def get_fixture_path(filename): 'tags': ['test:tag'], } -COMPOSE_FILE = os.path.join(HERE, 'docker', 'docker-compose.yaml') +COMPOSE_FILE = os.path.join(HERE, 'compose', 'docker-compose.yaml') STANDALONE_TEST_METRICS = { 'milvus.go.gc_duration_seconds.quantile': 'gauge', diff --git a/milvus/tests/compose/Dockerfile b/milvus/tests/compose/Dockerfile new file mode 100644 index 0000000000000..238dfdedd6d6d --- /dev/null +++ b/milvus/tests/compose/Dockerfile @@ -0,0 +1,13 @@ +# Use the Python slim image as a base +FROM python:3.9-slim + +# Set working directory +WORKDIR /app + +# Copy requirements file and install dependencies +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the Python script and healthcheck script +COPY script.py /app/script.py +COPY healthcheck_script.sh /app/healthcheck_script.sh diff --git a/milvus/tests/compose/docker-compose.yaml b/milvus/tests/compose/docker-compose.yaml new file mode 100644 index 0000000000000..f7225021924ff --- /dev/null +++ b/milvus/tests/compose/docker-compose.yaml @@ -0,0 +1,41 @@ +services: + milvus: + image: milvusdb/milvus + security_opt: + - seccomp:unconfined + environment: + - ETCD_USE_EMBED=true + - ETCD_DATA_DIR=/var/lib/milvus/etcd + - ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml + - COMMON_STORAGETYPE=local + volumes: + - /tmp/volumes/milvus:/var/lib/milvus + - ./embedEtcd.yaml:/milvus/configs/embedEtcd.yaml + - ./user.yaml:/milvus/configs/user.yaml + ports: + - "19530:19530" + - "9091:9091" + - "2379:2379" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + timeout: 20s + start_period: 90s + retries: 3 + command: milvus run standalone + + script-runner: + build: + context: . + dockerfile: Dockerfile + container_name: script-runner + depends_on: + milvus: + condition: service_healthy + volumes: + - ./script.py:/app/script.py + working_dir: /app + command: ["python", "script.py"] + environment: + - MILVUS_HOST=milvus + - MILVUS_PORT=19530 diff --git a/milvus/tests/compose/embedEtcd.yaml b/milvus/tests/compose/embedEtcd.yaml new file mode 100644 index 0000000000000..32954faa8b1fc --- /dev/null +++ b/milvus/tests/compose/embedEtcd.yaml @@ -0,0 +1,5 @@ +listen-client-urls: http://0.0.0.0:2379 +advertise-client-urls: http://0.0.0.0:2379 +quota-backend-bytes: 4294967296 +auto-compaction-mode: revision +auto-compaction-retention: '1000' diff --git a/milvus/tests/compose/requirements.txt b/milvus/tests/compose/requirements.txt new file mode 100644 index 0000000000000..1132a5bdcaca1 --- /dev/null +++ b/milvus/tests/compose/requirements.txt @@ -0,0 +1,2 @@ +numpy +pymilvus diff --git a/milvus/tests/compose/script.py b/milvus/tests/compose/script.py new file mode 100644 index 0000000000000..645aec1d4445e --- /dev/null +++ b/milvus/tests/compose/script.py @@ -0,0 +1,155 @@ +# hello_milvus.py demonstrates the basic operations of PyMilvus, a Python SDK of Milvus. +# 1. connect to Milvus +# 2. create collection +# 3. insert data +# 4. create index +# 5. search, query, and hybrid search on entities +# 6. delete entities by PK +# 7. drop collection +import time +import os + +import numpy as np +from pymilvus import (Collection, CollectionSchema, DataType, FieldSchema, + connections, utility) + + +num_entities, dim = 3000, 8 + +# Retrieve host and port from environment variables +MILVUS_HOST = os.getenv("MILVUS_HOST", "localhost") +MILVUS_PORT = os.getenv("MILVUS_PORT", "19530") + +################################################################################# +# 1. connect to Milvus +# Add a new connection alias `default` for Milvus server +connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT) + +has = utility.has_collection("hello_milvus") + +################################################################################# +# 2. create collection +# We're going to create a collection with 3 fields. +# +-+------------+------------+------------------+------------------------------+ +# | | field name | field type | other attributes | field description | +# +-+------------+------------+------------------+------------------------------+ +# |1| "pk" | VarChar | is_primary=True | "primary field" | +# | | | | auto_id=False | | +# +-+------------+------------+------------------+------------------------------+ +# |2| "random" | Double | | "a double field" | +# +-+------------+------------+------------------+------------------------------+ +# |3|"embeddings"| FloatVector| dim=8 | "float vector with dim 8" | +# +-+------------+------------+------------------+------------------------------+ +fields = [ + FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=100), + FieldSchema(name="random", dtype=DataType.DOUBLE), + FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim) +] + +schema = CollectionSchema(fields, "hello_milvus is the simplest demo to introduce the APIs") + +hello_milvus = Collection("hello_milvus", schema, consistency_level="Strong") + +################################################################################ +# 3. insert data +# We are going to insert 3000 rows of data into `hello_milvus` +# Data to be inserted must be organized in fields. +# +# The insert() method returns: +# - either automatically generated primary keys by Milvus if auto_id=True in the schema; +# - or the existing primary key field from the entities if auto_id=False in the schema. + +rng = np.random.default_rng(seed=19530) +entities = [ + # provide the pk field because `auto_id` is set to False + [str(i) for i in range(num_entities)], + rng.random(num_entities).tolist(), # field random, only supports list + rng.random((num_entities, dim), np.float32), # field embeddings, supports numpy.ndarray and list +] + +insert_result = hello_milvus.insert(entities) + +row = { + "pk": "19530", + "random": 0.5, + "embeddings": rng.random((1, dim), np.float32)[0] +} +hello_milvus.insert(row) + +hello_milvus.flush() + +################################################################################ +# 4. create index +# We are going to create an IVF_FLAT index for hello_milvus collection. +# create_index() can only be applied to `FloatVector` and `BinaryVector` fields. +index = { + "index_type": "IVF_FLAT", + "metric_type": "L2", + "params": {"nlist": 128}, +} + +hello_milvus.create_index("embeddings", index) + +################################################################################ +# 5. search, query, and hybrid search +# After data were inserted into Milvus and indexed, you can perform: +# - search based on vector similarity +# - query based on scalar filtering(boolean, int, etc.) +# - hybrid search based on vector similarity and scalar filtering. +# + +# Before conducting a search or a query, you need to load the data in `hello_milvus` into memory. +hello_milvus.load() + +# ----------------------------------------------------------------------------- +# search based on vector similarity +vectors_to_search = entities[-1][-2:] +search_params = { + "metric_type": "L2", + "params": {"nprobe": 10}, +} + +start_time = time.time() +result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, output_fields=["random"]) +end_time = time.time() + +# ----------------------------------------------------------------------------- +# query based on scalar filtering(boolean, int, etc.) + +start_time = time.time() +result = hello_milvus.query(expr="random > 0.5", output_fields=["random", "embeddings"]) +end_time = time.time() + +# ----------------------------------------------------------------------------- +# pagination +r1 = hello_milvus.query(expr="random > 0.5", limit=4, output_fields=["random"]) +r2 = hello_milvus.query(expr="random > 0.5", offset=1, limit=3, output_fields=["random"]) + + +# ----------------------------------------------------------------------------- +# hybrid search + +start_time = time.time() +result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, expr="random > 0.5", output_fields=["random"]) +end_time = time.time() + +############################################################################### +# 6. delete entities by PK +# You can delete entities by their PK values using boolean expressions. +ids = insert_result.primary_keys + +expr = f'pk in ["{ids[0]}" , "{ids[1]}"]' + +result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"]) + +hello_milvus.delete(expr) + +result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"]) + + +############################################################################### +# 7. drop collection +# Finally, drop the hello_milvus collection +utility.drop_collection("hello_milvus") + +print("Setup complete") diff --git a/milvus/tests/compose/user.yaml b/milvus/tests/compose/user.yaml new file mode 100644 index 0000000000000..8d312694bb2c8 --- /dev/null +++ b/milvus/tests/compose/user.yaml @@ -0,0 +1 @@ +# Extra config to override default milvus.yaml diff --git a/milvus/tests/conftest.py b/milvus/tests/conftest.py index c752c4ca012ec..7096c290c3265 100644 --- a/milvus/tests/conftest.py +++ b/milvus/tests/conftest.py @@ -11,13 +11,15 @@ from . import common +SCRIPT_COMPLETION_STR = "Setup complete" + @pytest.fixture(scope='session') def dd_environment(): compose_file = common.COMPOSE_FILE conditions = [ - CheckDockerLogs(identifier='caddy', patterns=['server running']), CheckEndpoints(common.MOCKED_INSTANCE["openmetrics_endpoint"]), + CheckDockerLogs("script-runner", SCRIPT_COMPLETION_STR) ] logging.info(conditions) with docker_run(compose_file, conditions=conditions): diff --git a/milvus/tests/docker/Caddyfile b/milvus/tests/docker/Caddyfile deleted file mode 100644 index 96d00534e6cb3..0000000000000 --- a/milvus/tests/docker/Caddyfile +++ /dev/null @@ -1,4 +0,0 @@ -:9091 { - root * /metrics - file_server -} diff --git a/milvus/tests/docker/docker-compose.yaml b/milvus/tests/docker/docker-compose.yaml deleted file mode 100644 index 8bb27b6a4468a..0000000000000 --- a/milvus/tests/docker/docker-compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - -# Caddy will create host a site with the metrics to exposes - caddy: - image: caddy:2.7 - container_name: caddy - ports: - - "9091:9091" - volumes: - - ./Caddyfile:/etc/caddy/Caddyfile - - ../fixtures/milvus_payload.txt:/metrics/metrics diff --git a/milvus/tests/test_e2e.py b/milvus/tests/test_e2e.py new file mode 100644 index 0000000000000..a6ccb89fdb8dd --- /dev/null +++ b/milvus/tests/test_e2e.py @@ -0,0 +1,20 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.constants import ServiceCheck + +from . import common + + +@pytest.mark.e2e +def test_check_milvus_e2e(dd_agent_check, instance): + aggregator = dd_agent_check(instance, rate=True) + + for metric, _ in common.STANDALONE_TEST_METRICS.items(): + if metric == 'milvus.datacoord.import_tasks': # this metric needs a more complex setup to appear + continue + aggregator.assert_metric(name=metric) + + aggregator.assert_service_check('milvus.openmetrics.health', ServiceCheck.OK) From d28e6bbe9dcc295230182c687f21f0f605c36025 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 16 Dec 2024 11:08:45 +0100 Subject: [PATCH 20/31] Run lint --- milvus/datadog_checks/milvus/metrics.py | 2 +- milvus/tests/common.py | 2 +- milvus/tests/compose/script.py | 20 ++++++++------------ milvus/tests/conftest.py | 2 +- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py index 6f70c7e42ab59..d3a9a434209f6 100644 --- a/milvus/datadog_checks/milvus/metrics.py +++ b/milvus/datadog_checks/milvus/metrics.py @@ -218,7 +218,7 @@ 'raw_compute_cnt': 'raw_compute_cnt', 're_search_cnt': 're_search_cnt', 'search_latency': 'search.latency', - 'search_topk': 'search.topk' + 'search_topk': 'search.topk', } RENAME_LABELS_MAP = { diff --git a/milvus/tests/common.py b/milvus/tests/common.py index 1e84b66583a8a..7bf28a3e887e7 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -390,7 +390,7 @@ def get_fixture_path(filename): 'milvus.search.topk.sum': 'monotonic_count', 'milvus.search.topk.count': 'monotonic_count', 'milvus.search.topk.bucket': 'monotonic_count', - } +} OTHER_TEST_METRICS = { 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', diff --git a/milvus/tests/compose/script.py b/milvus/tests/compose/script.py index 645aec1d4445e..c525220925a11 100644 --- a/milvus/tests/compose/script.py +++ b/milvus/tests/compose/script.py @@ -6,13 +6,11 @@ # 5. search, query, and hybrid search on entities # 6. delete entities by PK # 7. drop collection -import time import os +import time import numpy as np -from pymilvus import (Collection, CollectionSchema, DataType, FieldSchema, - connections, utility) - +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility num_entities, dim = 3000, 8 @@ -43,7 +41,7 @@ fields = [ FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=100), FieldSchema(name="random", dtype=DataType.DOUBLE), - FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim) + FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim), ] schema = CollectionSchema(fields, "hello_milvus is the simplest demo to introduce the APIs") @@ -64,16 +62,12 @@ # provide the pk field because `auto_id` is set to False [str(i) for i in range(num_entities)], rng.random(num_entities).tolist(), # field random, only supports list - rng.random((num_entities, dim), np.float32), # field embeddings, supports numpy.ndarray and list + rng.random((num_entities, dim), np.float32), # field embeddings, supports numpy.ndarray and list ] insert_result = hello_milvus.insert(entities) -row = { - "pk": "19530", - "random": 0.5, - "embeddings": rng.random((1, dim), np.float32)[0] -} +row = {"pk": "19530", "random": 0.5, "embeddings": rng.random((1, dim), np.float32)[0]} hello_milvus.insert(row) hello_milvus.flush() @@ -130,7 +124,9 @@ # hybrid search start_time = time.time() -result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, expr="random > 0.5", output_fields=["random"]) +result = hello_milvus.search( + vectors_to_search, "embeddings", search_params, limit=3, expr="random > 0.5", output_fields=["random"] +) end_time = time.time() ############################################################################### diff --git a/milvus/tests/conftest.py b/milvus/tests/conftest.py index 7096c290c3265..ab0a5dc8b52a7 100644 --- a/milvus/tests/conftest.py +++ b/milvus/tests/conftest.py @@ -19,7 +19,7 @@ def dd_environment(): compose_file = common.COMPOSE_FILE conditions = [ CheckEndpoints(common.MOCKED_INSTANCE["openmetrics_endpoint"]), - CheckDockerLogs("script-runner", SCRIPT_COMPLETION_STR) + CheckDockerLogs("script-runner", SCRIPT_COMPLETION_STR), ] logging.info(conditions) with docker_run(compose_file, conditions=conditions): From 17d286746dd25aa0deeffedf923c53e7e27b80af Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 16 Dec 2024 11:26:38 +0100 Subject: [PATCH 21/31] Fix CI --- .codecov.yml | 9 + .github/workflows/test-all.yml | 20 + milvus/changelog.d/{1.added => 19055.added} | 0 .../milvus/data/conf.yaml.example | 2 +- milvus/metadata.csv | 774 +++++++++--------- milvus/tests/compose/Dockerfile | 3 - milvus/tests/compose/requirements.txt | 4 +- 7 files changed, 419 insertions(+), 393 deletions(-) rename milvus/changelog.d/{1.added => 19055.added} (100%) diff --git a/.codecov.yml b/.codecov.yml index 12d18bf18d059..9aef58bee0c1c 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -410,6 +410,10 @@ coverage: target: 75 flags: - mesos_master + Milvus: + target: 75 + flags: + - milvus MongoDB: target: 75 flags: @@ -1254,6 +1258,11 @@ flags: paths: - mesos_slave/datadog_checks/mesos_slave - mesos_slave/tests + milvus: + carryforward: true + paths: + - milvus/datadog_checks/milvus + - milvus/tests mongo: carryforward: true paths: diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 53fc9b5f077c9..b32d88a9dc0c6 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -2494,6 +2494,26 @@ jobs: minimum-base-package: ${{ inputs.minimum-base-package }} pytest-args: ${{ inputs.pytest-args }} secrets: inherit + j062aeb0: + uses: ./.github/workflows/test-target.yml + with: + job-name: Milvus + target: milvus + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + python-version: "${{ inputs.python-version }}" + standard: ${{ inputs.standard }} + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + minimum-base-package: ${{ inputs.minimum-base-package }} + pytest-args: ${{ inputs.pytest-args }} + secrets: inherit j91231ff: uses: ./.github/workflows/test-target.yml with: diff --git a/milvus/changelog.d/1.added b/milvus/changelog.d/19055.added similarity index 100% rename from milvus/changelog.d/1.added rename to milvus/changelog.d/19055.added diff --git a/milvus/datadog_checks/milvus/data/conf.yaml.example b/milvus/datadog_checks/milvus/data/conf.yaml.example index 30d8e0cf300cb..b275903aba352 100644 --- a/milvus/datadog_checks/milvus/data/conf.yaml.example +++ b/milvus/datadog_checks/milvus/data/conf.yaml.example @@ -180,7 +180,7 @@ instances: ## @param collect_histogram_buckets - boolean - optional - default: true ## Whether or not to send histogram buckets. # - # collect_histogram_buckets: false + # collect_histogram_buckets: true ## @param non_cumulative_histogram_buckets - boolean - optional - default: false ## Whether or not histogram buckets are non-cumulative and to come with a `lower_bound` tag. diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 14e3ec6cdeca5..d37fc79da2c67 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,387 +1,387 @@ -metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -milvus.build_info,gauge,,,,Build information of milvus,0,milvus,, -milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,, -milvus.cgo.cgo_duration_seconds.sum,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_duration_seconds.count,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_duration_seconds.bucket,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Duration of cgo call in queue.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds.count,count,,second,,Duration of cgo call in queue.,0,milvus,, -milvus.cgo.cgo_queue_duration_seconds.bucket,count,,second,,Duration of cgo call in queue.,0,milvus,, -milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,, -milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,, -milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,, -milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,, -milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,, -milvus.datacoord.index.req.count,count,,,,number of building index requests,0,milvus,, -milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,, -milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,, -milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,, -milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,, -milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,, -milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,latency of task execute operation,0,milvus,, -milvus.datacoord.task_execute_max_latency.count,count,,millisecond,,latency of task execute operation,0,milvus,, -milvus.datacoord.task_execute_max_latency.bucket,count,,millisecond,,latency of task execute operation,0,milvus,, -milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,, -milvus.datanode.autoflush_buffer_op.count,count,,,,count of auto flush buffer operations,0,milvus,, -milvus.datanode.consume.bytes.count,count,,,,,0,milvus,, -milvus.datanode.consume.msg.count,count,,,,count of consumed msg,0,milvus,, -milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,latency of encode buffer data,0,milvus,, -milvus.datanode.encode_buffer_latency.count,count,,millisecond,,latency of encode buffer data,0,milvus,, -milvus.datanode.encode_buffer_latency.bucket,count,,millisecond,,latency of encode buffer data,0,milvus,, -milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.datanode.flush.buffer_op.count,count,,,,count of flush buffer operations,0,milvus,, -milvus.datanode.flush.req.count,count,,,,count of flush request,0,milvus,, -milvus.datanode.flushed_data.rows.count,count,,,,num of rows flushed to storage,0,milvus,, -milvus.datanode.flushed_data.size.count,count,,byte,,byte size of data flushed to storage,0,milvus,, -milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus.datanode.msg.rows.count,count,,,,count of rows consumed from msgStream,0,milvus,, -milvus.datanode.save_latency.sum,count,,millisecond,,latency of saving flush data to storage,0,milvus,, -milvus.datanode.save_latency.count,count,,millisecond,,latency of saving flush data to storage,0,milvus,, -milvus.datanode.save_latency.bucket,count,,millisecond,,latency of saving flush data to storage,0,milvus,, -milvus.flushed_segment_file_num.sum,count,,,,the num of files for flushed segment,0,milvus,, -milvus.flushed_segment_file_num.count,count,,,,the num of files for flushed segment,0,milvus,, -milvus.flushed_segment_file_num.bucket,count,,,,the num of files for flushed segment,0,milvus,, -milvus.indexnode.build_index_latency.sum,count,,millisecond,,latency of build index for segment,0,milvus,, -milvus.indexnode.build_index_latency.count,count,,millisecond,,latency of build index for segment,0,milvus,, -milvus.indexnode.build_index_latency.bucket,count,,millisecond,,latency of build index for segment,0,milvus,, -milvus.indexnode.encode_index_latency.sum,count,,millisecond,,latency of encoding the index file,0,milvus,, -milvus.indexnode.encode_index_latency.count,count,,millisecond,,latency of encoding the index file,0,milvus,, -milvus.indexnode.encode_index_latency.bucket,count,,millisecond,,latency of encoding the index file,0,milvus,, -milvus.indexnode.index.task.count,count,,,,number of tasks that index node received,0,milvus,, -milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,latency of index task in queue,0,milvus,, -milvus.indexnode.index.task_latency_in_queue.count,count,,millisecond,,latency of index task in queue,0,milvus,, -milvus.indexnode.index.task_latency_in_queue.bucket,count,,millisecond,,latency of index task in queue,0,milvus,, -milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.knowhere_build_index_latency.count,count,,millisecond,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.knowhere_build_index_latency.bucket,count,,millisecond,,latency of building the index by knowhere,0,milvus,, -milvus.indexnode.save_index_latency.sum,count,,millisecond,,latency of saving the index file,0,milvus,, -milvus.indexnode.save_index_latency.count,count,,millisecond,,latency of saving the index file,0,milvus,, -milvus.indexnode.save_index_latency.bucket,count,,millisecond,,latency of saving the index file,0,milvus,, -milvus.meta.kv_size.sum,count,,,,kv size stats,0,milvus,, -milvus.meta.kv_size.count,count,,,,kv size stats,0,milvus,, -milvus.meta.kv_size.bucket,count,,,,kv size stats,0,milvus,, -milvus.meta.op.count,count,,,,count of meta operation,0,milvus,, -milvus.meta.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.meta.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.meta.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,, -milvus.msgstream.op.count,count,,,,count of stream message operation,0,milvus,, -milvus.msgstream.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.msgstream.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.msgstream.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,, -milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,, -milvus.proxy.apply.pk_latency.sum,count,,millisecond,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.pk_latency.count,count,,millisecond,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.pk_latency.bucket,count,,millisecond,,latency that apply primary key,0,milvus,, -milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.apply.timestamp_latency.count,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.apply.timestamp_latency.bucket,count,,millisecond,,latency that proxy apply timestamp,0,milvus,, -milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.proxy.assign_segmentID_latency.count,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.proxy.assign_segmentID_latency.bucket,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,, -milvus.proxy.cache.hit.count,count,,,,count of cache hits/miss,0,milvus,, -milvus.proxy.cache.update_latency.sum,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, -milvus.proxy.cache.update_latency.count,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, -milvus.proxy.cache.update_latency.bucket,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,, -milvus.proxy.delete_vectors.count,count,,,,counter of vectors successfully deleted,0,milvus,, -milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,, -milvus.proxy.mutation_send_latency.sum,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.proxy.mutation_send_latency.count,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.proxy.mutation_send_latency.bucket,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,, -milvus.proxy.rate_limit_req.count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.report_value.count,count,,,,report value about the request,0,milvus,, -milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,, -milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.in_queue_latency.count,count,,millisecond,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.in_queue_latency.bucket,count,,millisecond,,latency which request waits in the queue,0,milvus,, -milvus.proxy.req.latency.sum,count,,millisecond,,latency of each request,0,milvus,, -milvus.proxy.req.latency.count,count,,millisecond,,latency of each request,0,milvus,, -milvus.proxy.req.latency.bucket,count,,millisecond,,latency of each request,0,milvus,, -milvus.proxy.send_bytes.count,count,,,,count of bytes sent back to sdk,0,milvus,, -milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.decode_result_latency.count,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.decode_result_latency.bucket,count,,millisecond,,latency that proxy decodes the search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency.count,count,,millisecond,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.reduce_result_latency.bucket,count,,millisecond,,latency that proxy reduces search result,0,milvus,, -milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,latency that proxy waits for the result,0,milvus,, -milvus.proxy.sq.wait_result_latency.count,count,,millisecond,,latency that proxy waits for the result,0,milvus,, -milvus.proxy.sq.wait_result_latency.bucket,count,,millisecond,,latency that proxy waits for the result,0,milvus,, -milvus.proxy.sync_segment_request_length.sum,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.proxy.sync_segment_request_length.count,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.proxy.sync_segment_request_length.bucket,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,, -milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,, -milvus.querycoord.load.latency.sum,count,,millisecond,,latency of load the entire collection,0,milvus,, -milvus.querycoord.load.latency.count,count,,millisecond,,latency of load the entire collection,0,milvus,, -milvus.querycoord.load.latency.bucket,count,,millisecond,,latency of load the entire collection,0,milvus,, -milvus.querycoord.load.req.count,count,,,,count of load request,0,milvus,, -milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,, -milvus.querycoord.release.latency.sum,count,,millisecond,,latency of release request,0,milvus,, -milvus.querycoord.release.latency.count,count,,millisecond,,latency of release request,0,milvus,, -milvus.querycoord.release.latency.bucket,count,,millisecond,,latency of release request,0,milvus,, -milvus.querycoord.release.req.count,count,,,,count of release request,0,milvus,, -milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,, -milvus.querynode.apply_bf_latency.sum,count,,millisecond,,apply bf cost in ms,0,milvus,, -milvus.querynode.apply_bf_latency.count,count,,millisecond,,apply bf cost in ms,0,milvus,, -milvus.querynode.apply_bf_latency.bucket,count,,millisecond,,apply bf cost in ms,0,milvus,, -milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,, -milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,, -milvus.querynode.consume.msg.count,count,,,,count of consumed msg,0,milvus,, -milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,number of bytes evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.duration.count,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration.count,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.global_duration.bucket,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.evict.count,count,,,,number of segments evicted from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.bytes.count,count,,byte,,number of bytes loaded from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.duration.count,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration.count,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.global_duration.bucket,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,, -milvus.querynode.disk.cache.load.count,count,,,,number of segments loaded from disk cache,0,milvus,, -milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,, -milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,, -milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,, -milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,, -milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,, -milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,, -milvus.querynode.forward_delete_latency.sum,count,,millisecond,,forward delete cost in ms,0,milvus,, -milvus.querynode.forward_delete_latency.count,count,,millisecond,,forward delete cost in ms,0,milvus,, -milvus.querynode.forward_delete_latency.bucket,count,,millisecond,,forward delete cost in ms,0,milvus,, -milvus.querynode.load.index_latency.sum,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus.querynode.load.index_latency.count,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus.querynode.load.index_latency.bucket,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,, -milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,, -milvus.querynode.load.segment.latency.sum,count,,millisecond,,latency of load per segment,0,milvus,, -milvus.querynode.load.segment.latency.count,count,,millisecond,,latency of load per segment,0,milvus,, -milvus.querynode.load.segment.latency.bucket,count,,millisecond,,latency of load per segment,0,milvus,, -milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,, -milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,, -milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,process insert or delete cost in ms,0,milvus,, -milvus.querynode.process_insert_or_delete_latency.count,count,,millisecond,,process insert or delete cost in ms,0,milvus,, -milvus.querynode.process_insert_or_delete_latency.bucket,count,,millisecond,,process insert or delete cost in ms,0,milvus,, -milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,, -milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,, -milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,, -milvus.querynode.search.group.nq.sum,count,,,,the number of queries of each grouped search task,0,milvus,, -milvus.querynode.search.group.nq.count,count,,,,the number of queries of each grouped search task,0,milvus,, -milvus.querynode.search.group.nq.bucket,count,,,,the number of queries of each grouped search task,0,milvus,, -milvus.querynode.search.group.size.sum,count,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.querynode.search.group.size.count,count,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.querynode.search.group.size.bucket,count,,,,the number of tasks of each grouped search task,0,milvus,, -milvus.querynode.search.group.topk.sum,count,,,,the topK of each grouped search task,0,milvus,, -milvus.querynode.search.group.topk.count,count,,,,the topK of each grouped search task,0,milvus,, -milvus.querynode.search.group.topk.bucket,count,,,,the topK of each grouped search task,0,milvus,, -milvus.querynode.search.nq.sum,count,,,,the number of queries of each search task,0,milvus,, -milvus.querynode.search.nq.count,count,,,,the number of queries of each search task,0,milvus,, -milvus.querynode.search.nq.bucket,count,,,,the number of queries of each search task,0,milvus,, -milvus.querynode.search.topk.sum,count,,,,the top of each search task,0,milvus,, -milvus.querynode.search.topk.count,count,,,,the top of each search task,0,milvus,, -milvus.querynode.search.topk.bucket,count,,,,the top of each search task,0,milvus,, -milvus.querynode.segment.access.duration.count,count,,millisecond,,total time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,global time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration.count,count,,millisecond,,global time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.global_duration.bucket,count,,millisecond,,global time cost of accessing segments,0,milvus,, -milvus.querynode.segment.access.count,count,,,,number of segments accessed,0,milvus,, -milvus.querynode.segment.access.wait_cache.duration.count,count,,millisecond,,total time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration.count,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,millisecond,,global time cost of waiting for loading access,0,milvus,, -milvus.querynode.segment.access.wait_cache.count,count,,,,number of segments waiting for loading access,0,milvus,, -milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,one vector's search latency per segment,0,milvus,, -milvus.querynode.segment.latency_per_vector.count,count,,millisecond,,one vector's search latency per segment,0,milvus,, -milvus.querynode.segment.latency_per_vector.bucket,count,,millisecond,,one vector's search latency per segment,0,milvus,, -milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,, -milvus.querynode.sq.core_latency.sum,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.core_latency.count,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.core_latency.bucket,count,,millisecond,,latency of search or query latency in segcore,0,milvus,, -milvus.querynode.sq.queue.latency.sum,count,,millisecond,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.latency.count,count,,millisecond,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.latency.bucket,count,,millisecond,,latency of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency.count,count,,millisecond,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.queue.user_latency.bucket,count,,millisecond,,latency per user of search or query in queue,0,milvus,, -milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,latency of reduce search or query result,0,milvus,, -milvus.querynode.sq.reduce_latency.count,count,,millisecond,,latency of reduce search or query result,0,milvus,, -milvus.querynode.sq.reduce_latency.bucket,count,,millisecond,,latency of reduce search or query result,0,milvus,, -milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,, -milvus.querynode.sq.req.latency.sum,count,,millisecond,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.req.latency.count,count,,millisecond,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.req.latency.bucket,count,,millisecond,,latency of Search or query requests,0,milvus,, -milvus.querynode.sq.segment_latency.sum,count,,millisecond,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.segment_latency.count,count,,millisecond,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.segment_latency.bucket,count,,millisecond,,latency of search or query per segment,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency.count,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, -milvus.querynode.sq.wait_tsafe_latency.bucket,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,, -milvus.querynode.wait_processing_msg,gauge,,,.count,count of wait processing msg,0,milvus,, -milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,latency of watch dml channel,0,milvus,, -milvus.querynode.watch_dml_channel_latency.count,count,,millisecond,,latency of watch dml channel,0,milvus,, -milvus.querynode.watch_dml_channel_latency.bucket,count,,millisecond,,latency of watch dml channel,0,milvus,, -milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,, -milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,, -milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency.count,count,,millisecond,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency.bucket,count,,millisecond,,latency of each DDL operations,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue.count,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, -milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,millisecond,,latency of each DDL operations in queue,0,milvus,, -milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,, -milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,, -milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,, -milvus.rootcoord.id_alloc.count,count,,,,count of ID allocated,0,milvus,, -milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,, -milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,, -milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,, -milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,, -milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,, -milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,, -milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,, -milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, -milvus.rootcoord.sync_timetick_latency.count,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, -milvus.rootcoord.sync_timetick_latency.bucket,count,,millisecond,,latency of synchronizing timetick message,0,milvus,, -milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,, -milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,, -milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,, -milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,, -milvus.storage.op.count,count,,,,count of persistent data operation,0,milvus,, -milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,, -milvus.bf_search_cnt.sum,count,,,,number of bf search per request,0,milvus,, -milvus.bf_search_cnt.count,count,,,,number of bf search per request,0,milvus,, -milvus.bf_search_cnt.bucket,count,,,,number of bf search per request,0,milvus,, -milvus.bitset_ratio.sum,count,,,,bitset ratio,0,milvus,, -milvus.bitset_ratio.count,count,,,,bitset ratio,0,milvus,, -milvus.bitset_ratio.bucket,count,,,,bitset ratio,0,milvus,, -milvus.build_latency.sum,count,,,,index build latency (s),0,milvus,, -milvus.build_latency.count,count,,,,index build latency (s),0,milvus,, -milvus.build_latency.bucket,count,,,,index build latency (s),0,milvus,, -milvus.cache_hit_cnt.sum,count,,,,cache hit cnt per request,0,milvus,, -milvus.cache_hit_cnt.count,count,,,,cache hit cnt per request,0,milvus,, -milvus.cache_hit_cnt.bucket,count,,,,cache hit cnt per request,0,milvus,, -milvus.diskann.range_search_iters.sum,count,,,,DISKANN range search iterations,0,milvus,, -milvus.diskann.range_search_iters.count,count,,,,DISKANN range search iterations,0,milvus,, -milvus.diskann.range_search_iters.bucket,count,,,,DISKANN range search iterations,0,milvus,, -milvus.diskann.search_hops.sum,count,,,,DISKANN search hops,0,milvus,, -milvus.diskann.search_hops.count,count,,,,DISKANN search hops,0,milvus,, -milvus.diskann.search_hops.bucket,count,,,,DISKANN search hops,0,milvus,, -milvus.diskann_bitset_ratio.sum,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, -milvus.diskann_bitset_ratio.count,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, -milvus.diskann_bitset_ratio.bucket,count,,,,DISKANN bitset ratio for search and range search,0,milvus,, -milvus.exec_latency.sum,count,,,,execute latency per request,0,milvus,, -milvus.exec_latency.count,count,,,,execute latency per request,0,milvus,, -milvus.exec_latency.bucket,count,,,,execute latency per request,0,milvus,, -milvus.filter.connectivity_ratio.sum,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, -milvus.filter.connectivity_ratio.count,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, -milvus.filter.connectivity_ratio.bucket,count,,,,avg connectivity ratio set under filtering per request,0,milvus,, -milvus.filter.mv.activated_fields_cnt.sum,count,,,,avg mv activated fields per request,0,milvus,, -milvus.filter.mv.activated_fields_cnt.count,count,,,,avg mv activated fields per request,0,milvus,, -milvus.filter.mv.activated_fields_cnt.bucket,count,,,,avg mv activated fields per request,0,milvus,, -milvus.filter.mv.change_base_cnt.sum,count,,,,mv change base cnt per request,0,milvus,, -milvus.filter.mv.change_base_cnt.count,count,,,,mv change base cnt per request,0,milvus,, -milvus.filter.mv.change_base_cnt.bucket,count,,,,mv change base cnt per request,0,milvus,, -milvus.filter.mv.only_cnt.sum,count,,,,mv only cnt per request,0,milvus,, -milvus.filter.mv.only_cnt.count,count,,,,mv only cnt per request,0,milvus,, -milvus.filter.mv.only_cnt.bucket,count,,,,mv only cnt per request,0,milvus,, -milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, -milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, -milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,, -milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,, -milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,, -milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,, -milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,, -milvus.go.memstats.alloc_bytes.count,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,, -milvus.go.memstats.buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,, -milvus.go.memstats.frees.count,count,,,,Total number of frees.,0,milvus,, -milvus.go.memstats.gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,, -milvus.go.memstats.heap.alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,, -milvus.go.memstats.heap.idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,, -milvus.go.memstats.heap.inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,, -milvus.go.memstats.heap.objects,gauge,,,,Number of allocated objects.,0,milvus,, -milvus.go.memstats.heap.released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,, -milvus.go.memstats.heap.sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,, -milvus.go.memstats.last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,, -milvus.go.memstats.lookups.count,count,,,,Total number of pointer lookups.,0,milvus,, -milvus.go.memstats.mallocs.count,count,,,,Total number of mallocs.,0,milvus,, -milvus.go.memstats.mcache.inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,, -milvus.go.memstats.mcache.sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,, -milvus.go.memstats.mspan.inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,, -milvus.go.memstats.mspan.sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,, -milvus.go.memstats.next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,, -milvus.go.memstats.other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,, -milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,, -milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,, -milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,, -milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,, -milvus.graph_search_cnt.sum,count,,,,number of graph search per request,0,milvus,, -milvus.graph_search_cnt.count,count,,,,number of graph search per request,0,milvus,, -milvus.graph_search_cnt.bucket,count,,,,number of graph search per request,0,milvus,, -milvus.hnsw.bitset_ratio.sum,count,,,,HNSW bitset ratio for search and range search,0,milvus,, -milvus.hnsw.bitset_ratio.count,count,,,,HNSW bitset ratio for search and range search,0,milvus,, -milvus.hnsw.bitset_ratio.bucket,count,,,,HNSW bitset ratio for search and range search,0,milvus,, -milvus.hnsw.search_hops.sum,count,,,,HNSW search hops in layer 0,0,milvus,, -milvus.hnsw.search_hops.count,count,,,,HNSW search hops in layer 0,0,milvus,, -milvus.hnsw.search_hops.bucket,count,,,,HNSW search hops in layer 0,0,milvus,, -milvus.internal.core_search_latency.sum,count,,,,[cpp]latency(us) of search on segment,0,milvus,, -milvus.internal.core_search_latency.count,count,,,,[cpp]latency(us) of search on segment,0,milvus,, -milvus.internal.core_search_latency.bucket,count,,,,[cpp]latency(us) of search on segment,0,milvus,, -milvus.internal.mmap.allocated_space_bytes.sum,count,,,,[cpp]mmap allocated space stats,0,milvus,, -milvus.internal.mmap.allocated_space_bytes.count,count,,,,[cpp]mmap allocated space stats,0,milvus,, -milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,[cpp]mmap allocated space stats,0,milvus,, -milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,, -milvus.internal.storage.kv_size.sum,count,,,,[cpp]kv size stats,0,milvus,, -milvus.internal.storage.kv_size.count,count,,,,[cpp]kv size stats,0,milvus,, -milvus.internal.storage.kv_size.bucket,count,,,,[cpp]kv size stats,0,milvus,, -milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segment,0,milvus,, -milvus.internal.storage.load_duration.count,count,,,,[cpp]durations of load segment,0,milvus,, -milvus.internal.storage.load_duration.bucket,count,,,,[cpp]durations of load segment,0,milvus,, -milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,, -milvus.internal.storage.request_latency.sum,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, -milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, -milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,, -milvus.io_cnt.sum,count,,,,io cnt per request,0,milvus,, -milvus.io_cnt.count,count,,,,io cnt per request,0,milvus,, -milvus.io_cnt.bucket,count,,,,io cnt per request,0,milvus,, -milvus.ivf_search_cnt.sum,count,,,,number of ivf search per request,0,milvus,, -milvus.ivf_search_cnt.count,count,,,,number of ivf search per request,0,milvus,, -milvus.ivf_search_cnt.bucket,count,,,,number of ivf search per request,0,milvus,, -milvus.load_latency.sum,count,,,,index load latency (ms),0,milvus,, -milvus.load_latency.count,count,,,,index load latency (ms),0,milvus,, -milvus.load_latency.bucket,count,,,,index load latency (ms),0,milvus,, -milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,, -milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,, -milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,, -milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,, -milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,, -milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,, -milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,, -milvus.quant.compute_cnt.sum,count,,,,quant compute cnt per request,0,milvus,, -milvus.quant.compute_cnt.count,count,,,,quant compute cnt per request,0,milvus,, -milvus.quant.compute_cnt.bucket,count,,,,quant compute cnt per request,0,milvus,, -milvus.queue.latency.sum,count,,,,queue latency per request,0,milvus,, -milvus.queue.latency.count,count,,,,queue latency per request,0,milvus,, -milvus.queue.latency.bucket,count,,,,queue latency per request,0,milvus,, -milvus.range_search_latency.sum,count,,,,range search latency (ms),0,milvus,, -milvus.range_search_latency.count,count,,,,range search latency (ms),0,milvus,, -milvus.range_search_latency.bucket,count,,,,range search latency (ms),0,milvus,, -milvus.raw_compute_cnt.sum,count,,,,raw compute cnt per request,0,milvus,, -milvus.raw_compute_cnt.count,count,,,,raw compute cnt per request,0,milvus,, -milvus.raw_compute_cnt.bucket,count,,,,raw compute cnt per request,0,milvus,, -milvus.re_search_cnt.sum,count,,,,number of fallback search per request,0,milvus,, -milvus.re_search_cnt.count,count,,,,number of fallback search per request,0,milvus,, -milvus.re_search_cnt.bucket,count,,,,number of fallback search per request,0,milvus,, -milvus.search.latency.sum,count,,,,search latency (ms),0,milvus,, -milvus.search.latency.count,count,,,,search latency (ms),0,milvus,, -milvus.search.latency.bucket,count,,,,search latency (ms),0,milvus,, -milvus.search.topk.sum,count,,,,search topk,0,milvus,, -milvus.search.topk.count,count,,,,search topk,0,milvus,, -milvus.search.topk.bucket,count,,,,search topk,0,milvus,, +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +milvus.bf_search_cnt.bucket,count,,,,number of bf search per request,0,milvus,,, +milvus.bf_search_cnt.count,count,,,,number of bf search per request,0,milvus,,, +milvus.bf_search_cnt.sum,count,,,,number of bf search per request,0,milvus,,, +milvus.bitset_ratio.bucket,count,,,,bitset ratio,0,milvus,,, +milvus.bitset_ratio.count,count,,,,bitset ratio,0,milvus,,, +milvus.bitset_ratio.sum,count,,,,bitset ratio,0,milvus,,, +milvus.build_info,gauge,,,,Build information of milvus,0,milvus,,, +milvus.build_latency.bucket,count,,,,index build latency (s),0,milvus,,, +milvus.build_latency.count,count,,,,index build latency (s),0,milvus,,, +milvus.build_latency.sum,count,,,,index build latency (s),0,milvus,,, +milvus.cache_hit_cnt.bucket,count,,,,cache hit cnt per request,0,milvus,,, +milvus.cache_hit_cnt.count,count,,,,cache hit cnt per request,0,milvus,,, +milvus.cache_hit_cnt.sum,count,,,,cache hit cnt per request,0,milvus,,, +milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.bucket,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.count,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.sum,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.bucket,count,,second,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.count,count,,second,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,,, +milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, +milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,,, +milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,,, +milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,,, +milvus.datacoord.index.req.count,count,,,,number of building index requests,0,milvus,,, +milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,,, +milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,,, +milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,,, +milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,,, +milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,,, +milvus.datacoord.task_execute_max_latency.bucket,count,,millisecond,,latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.count,count,,millisecond,,latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,latency of task execute operation,0,milvus,,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,,, +milvus.datanode.autoflush_buffer_op.count,count,,,,count of auto flush buffer operations,0,milvus,,, +milvus.datanode.consume.bytes.count,count,,,,,0,milvus,,, +milvus.datanode.consume.msg.count,count,,,,count of consumed msg,0,milvus,,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, +milvus.datanode.encode_buffer_latency.bucket,count,,millisecond,,latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.count,count,,millisecond,,latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,latency of encode buffer data,0,milvus,,, +milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,,, +milvus.datanode.flush.buffer_op.count,count,,,,count of flush buffer operations,0,milvus,,, +milvus.datanode.flush.req.count,count,,,,count of flush request,0,milvus,,, +milvus.datanode.flushed_data.rows.count,count,,,,num of rows flushed to storage,0,milvus,,, +milvus.datanode.flushed_data.size.count,count,,byte,,byte size of data flushed to storage,0,milvus,,, +milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, +milvus.datanode.msg.rows.count,count,,,,count of rows consumed from msgStream,0,milvus,,, +milvus.datanode.save_latency.bucket,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.count,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.sum,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, +milvus.diskann.range_search_iters.bucket,count,,,,DISKANN range search iterations,0,milvus,,, +milvus.diskann.range_search_iters.count,count,,,,DISKANN range search iterations,0,milvus,,, +milvus.diskann.range_search_iters.sum,count,,,,DISKANN range search iterations,0,milvus,,, +milvus.diskann.search_hops.bucket,count,,,,DISKANN search hops,0,milvus,,, +milvus.diskann.search_hops.count,count,,,,DISKANN search hops,0,milvus,,, +milvus.diskann.search_hops.sum,count,,,,DISKANN search hops,0,milvus,,, +milvus.diskann_bitset_ratio.bucket,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.count,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.sum,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.exec_latency.bucket,count,,,,execute latency per request,0,milvus,,, +milvus.exec_latency.count,count,,,,execute latency per request,0,milvus,,, +milvus.exec_latency.sum,count,,,,execute latency per request,0,milvus,,, +milvus.filter.connectivity_ratio.bucket,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.count,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.sum,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.bucket,count,,,,avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.count,count,,,,avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.sum,count,,,,avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.bucket,count,,,,mv change base cnt per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.count,count,,,,mv change base cnt per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.sum,count,,,,mv change base cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.bucket,count,,,,mv only cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.count,count,,,,mv only cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.sum,count,,,,mv only cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.flushed_segment_file_num.bucket,count,,,,the num of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.count,count,,,,the num of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.sum,count,,,,the num of files for flushed segment,0,milvus,,, +milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,,, +milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,,, +milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,,, +milvus.go.memstats.alloc_bytes.count,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,,, +milvus.go.memstats.buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,,, +milvus.go.memstats.frees.count,count,,,,Total number of frees.,0,milvus,,, +milvus.go.memstats.gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,,, +milvus.go.memstats.heap.alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,,, +milvus.go.memstats.heap.idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,,, +milvus.go.memstats.heap.inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,,, +milvus.go.memstats.heap.objects,gauge,,,,Number of allocated objects.,0,milvus,,, +milvus.go.memstats.heap.released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,,, +milvus.go.memstats.heap.sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,,, +milvus.go.memstats.last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,,, +milvus.go.memstats.lookups.count,count,,,,Total number of pointer lookups.,0,milvus,,, +milvus.go.memstats.mallocs.count,count,,,,Total number of mallocs.,0,milvus,,, +milvus.go.memstats.mcache.inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,,, +milvus.go.memstats.mcache.sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,,, +milvus.go.memstats.mspan.inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,,, +milvus.go.memstats.mspan.sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,,, +milvus.go.memstats.next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,,, +milvus.go.memstats.other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,,, +milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,,, +milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,,, +milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,,, +milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,,, +milvus.graph_search_cnt.bucket,count,,,,number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.count,count,,,,number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.sum,count,,,,number of graph search per request,0,milvus,,, +milvus.hnsw.bitset_ratio.bucket,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.count,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.sum,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.search_hops.bucket,count,,,,HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.count,count,,,,HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.sum,count,,,,HNSW search hops in layer 0,0,milvus,,, +milvus.indexnode.build_index_latency.bucket,count,,millisecond,,latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.count,count,,millisecond,,latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.sum,count,,millisecond,,latency of build index for segment,0,milvus,,, +milvus.indexnode.encode_index_latency.bucket,count,,millisecond,,latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.count,count,,millisecond,,latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.sum,count,,millisecond,,latency of encoding the index file,0,milvus,,, +milvus.indexnode.index.task.count,count,,,,number of tasks that index node received,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.bucket,count,,millisecond,,latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.count,count,,millisecond,,latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,latency of index task in queue,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.bucket,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.count,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.save_index_latency.bucket,count,,millisecond,,latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.count,count,,millisecond,,latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.sum,count,,millisecond,,latency of saving the index file,0,milvus,,, +milvus.internal.core_search_latency.bucket,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.count,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.sum,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,[cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.count,count,,,,[cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.sum,count,,,,[cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,,, +milvus.internal.storage.kv_size.bucket,count,,,,[cpp]kv size stats,0,milvus,,, +milvus.internal.storage.kv_size.count,count,,,,[cpp]kv size stats,0,milvus,,, +milvus.internal.storage.kv_size.sum,count,,,,[cpp]kv size stats,0,milvus,,, +milvus.internal.storage.load_duration.bucket,count,,,,[cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.count,count,,,,[cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,,, +milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.sum,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, +milvus.io_cnt.bucket,count,,,,io cnt per request,0,milvus,,, +milvus.io_cnt.count,count,,,,io cnt per request,0,milvus,,, +milvus.io_cnt.sum,count,,,,io cnt per request,0,milvus,,, +milvus.ivf_search_cnt.bucket,count,,,,number of ivf search per request,0,milvus,,, +milvus.ivf_search_cnt.count,count,,,,number of ivf search per request,0,milvus,,, +milvus.ivf_search_cnt.sum,count,,,,number of ivf search per request,0,milvus,,, +milvus.load_latency.bucket,count,,,,index load latency (ms),0,milvus,,, +milvus.load_latency.count,count,,,,index load latency (ms),0,milvus,,, +milvus.load_latency.sum,count,,,,index load latency (ms),0,milvus,,, +milvus.meta.kv_size.bucket,count,,,,kv size stats,0,milvus,,, +milvus.meta.kv_size.count,count,,,,kv size stats,0,milvus,,, +milvus.meta.kv_size.sum,count,,,,kv size stats,0,milvus,,, +milvus.meta.op.count,count,,,,count of meta operation,0,milvus,,, +milvus.meta.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.meta.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.meta.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,,, +milvus.msgstream.op.count,count,,,,count of stream message operation,0,milvus,,, +milvus.msgstream.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,,, +milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,,, +milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,,, +milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,,, +milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,,, +milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,,, +milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,,, +milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,,, +milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,,, +milvus.proxy.apply.pk_latency.bucket,count,,millisecond,,latency that apply primary key,0,milvus,,, +milvus.proxy.apply.pk_latency.count,count,,millisecond,,latency that apply primary key,0,milvus,,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,latency that apply primary key,0,milvus,,, +milvus.proxy.apply.timestamp_latency.bucket,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.apply.timestamp_latency.count,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.assign_segmentID_latency.bucket,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.assign_segmentID_latency.count,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.cache.hit.count,count,,,,count of cache hits/miss,0,milvus,,, +milvus.proxy.cache.update_latency.bucket,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.cache.update_latency.count,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.delete_vectors.count,count,,,,counter of vectors successfully deleted,0,milvus,,, +milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,,, +milvus.proxy.mutation_send_latency.bucket,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.mutation_send_latency.count,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.rate_limit_req.count,count,,,,count of operation executed,0,milvus,,, +milvus.proxy.report_value.count,count,,,,report value about the request,0,milvus,,, +milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,,, +milvus.proxy.req.in_queue_latency.bucket,count,,millisecond,,latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.count,count,,millisecond,,latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.latency.bucket,count,,millisecond,,latency of each request,0,milvus,,, +milvus.proxy.req.latency.count,count,,millisecond,,latency of each request,0,milvus,,, +milvus.proxy.req.latency.sum,count,,millisecond,,latency of each request,0,milvus,,, +milvus.proxy.send_bytes.count,count,,,,count of bytes sent back to sdk,0,milvus,,, +milvus.proxy.sq.decode_result_latency.bucket,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.decode_result_latency.count,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.count,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.bucket,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.count,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sync_segment_request_length.bucket,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.count,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.sum,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, +milvus.quant.compute_cnt.bucket,count,,,,quant compute cnt per request,0,milvus,,, +milvus.quant.compute_cnt.count,count,,,,quant compute cnt per request,0,milvus,,, +milvus.quant.compute_cnt.sum,count,,,,quant compute cnt per request,0,milvus,,, +milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,,, +milvus.querycoord.load.latency.bucket,count,,millisecond,,latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.count,count,,millisecond,,latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.sum,count,,millisecond,,latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.req.count,count,,,,count of load request,0,milvus,,, +milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,,, +milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,,, +milvus.querycoord.release.latency.bucket,count,,millisecond,,latency of release request,0,milvus,,, +milvus.querycoord.release.latency.count,count,,millisecond,,latency of release request,0,milvus,,, +milvus.querycoord.release.latency.sum,count,,millisecond,,latency of release request,0,milvus,,, +milvus.querycoord.release.req.count,count,,,,count of release request,0,milvus,,, +milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,,, +milvus.querynode.apply_bf_latency.bucket,count,,millisecond,,apply bf cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.count,count,,millisecond,,apply bf cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.sum,count,,millisecond,,apply bf cost in ms,0,milvus,,, +milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,,, +milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,,, +milvus.querynode.consume.msg.count,count,,,,count of consumed msg,0,milvus,,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, +milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,number of bytes evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.count,count,,,,number of segments evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.duration.count,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.bucket,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.count,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.bytes.count,count,,byte,,number of bytes loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.count,count,,,,number of segments loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.duration.count,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.bucket,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.count,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,,, +milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,,, +milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, +milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,,, +milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,,, +milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,,, +milvus.querynode.forward_delete_latency.bucket,count,,millisecond,,forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.count,count,,millisecond,,forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.sum,count,,millisecond,,forward delete cost in ms,0,milvus,,, +milvus.querynode.load.index_latency.bucket,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.count,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.sum,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,,, +milvus.querynode.load.segment.latency.bucket,count,,millisecond,,latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.count,count,,millisecond,,latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.sum,count,,millisecond,,latency of load per segment,0,milvus,,, +milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, +milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.bucket,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.count,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, +milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,,, +milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,,, +milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,,, +milvus.querynode.search.group.nq.bucket,count,,,,the number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.count,count,,,,the number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.sum,count,,,,the number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.bucket,count,,,,the number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.count,count,,,,the number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.sum,count,,,,the number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.bucket,count,,,,the topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.count,count,,,,the topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.sum,count,,,,the topK of each grouped search task,0,milvus,,, +milvus.querynode.search.nq.bucket,count,,,,the number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.count,count,,,,the number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.sum,count,,,,the number of queries of each search task,0,milvus,,, +milvus.querynode.search.topk.bucket,count,,,,the top of each search task,0,milvus,,, +milvus.querynode.search.topk.count,count,,,,the top of each search task,0,milvus,,, +milvus.querynode.search.topk.sum,count,,,,the top of each search task,0,milvus,,, +milvus.querynode.segment.access.count,count,,,,number of segments accessed,0,milvus,,, +milvus.querynode.segment.access.duration.count,count,,millisecond,,total time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.bucket,count,,millisecond,,global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.count,count,,millisecond,,global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.wait_cache.count,count,,,,number of segments waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.duration.count,count,,millisecond,,total time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.count,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.latency_per_vector.bucket,count,,millisecond,,one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.count,count,,millisecond,,one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,,, +milvus.querynode.sq.core_latency.bucket,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.count,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.sum,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.queue.latency.bucket,count,,millisecond,,latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.count,count,,millisecond,,latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.sum,count,,millisecond,,latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.bucket,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.count,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.reduce_latency.bucket,count,,millisecond,,latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.count,count,,millisecond,,latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,,, +milvus.querynode.sq.req.latency.bucket,count,,millisecond,,latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.count,count,,millisecond,,latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.sum,count,,millisecond,,latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.segment_latency.bucket,count,,millisecond,,latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.count,count,,millisecond,,latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.sum,count,,millisecond,,latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.bucket,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.count,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.wait_processing_msg,gauge,,,,count of wait processing msg,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,millisecond,,latency of watch dml channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.count,count,,millisecond,,latency of watch dml channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,latency of watch dml channel,0,milvus,,, +milvus.queue.latency.bucket,count,,,,queue latency per request,0,milvus,,, +milvus.queue.latency.count,count,,,,queue latency per request,0,milvus,,, +milvus.queue.latency.sum,count,,,,queue latency per request,0,milvus,,, +milvus.range_search_latency.bucket,count,,,,range search latency (ms),0,milvus,,, +milvus.range_search_latency.count,count,,,,range search latency (ms),0,milvus,,, +milvus.range_search_latency.sum,count,,,,range search latency (ms),0,milvus,,, +milvus.raw_compute_cnt.bucket,count,,,,raw compute cnt per request,0,milvus,,, +milvus.raw_compute_cnt.count,count,,,,raw compute cnt per request,0,milvus,,, +milvus.raw_compute_cnt.sum,count,,,,raw compute cnt per request,0,milvus,,, +milvus.re_search_cnt.bucket,count,,,,number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.count,count,,,,number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.sum,count,,,,number of fallback search per request,0,milvus,,, +milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,,, +milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,,, +milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.bucket,count,,millisecond,,latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.count,count,,millisecond,,latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.count,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,,, +milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,,, +milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,,, +milvus.rootcoord.id_alloc.count,count,,,,count of ID allocated,0,milvus,,, +milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,,, +milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,,, +milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,,, +milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, +milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,,, +milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.bucket,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.count,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,,, +milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,,, +milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,,, +milvus.search.latency.bucket,count,,,,search latency (ms),0,milvus,,, +milvus.search.latency.count,count,,,,search latency (ms),0,milvus,,, +milvus.search.latency.sum,count,,,,search latency (ms),0,milvus,,, +milvus.search.topk.bucket,count,,,,search topk,0,milvus,,, +milvus.search.topk.count,count,,,,search topk,0,milvus,,, +milvus.search.topk.sum,count,,,,search topk,0,milvus,,, +milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,,, +milvus.storage.op.count,count,,,,count of persistent data operation,0,milvus,,, +milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,,, diff --git a/milvus/tests/compose/Dockerfile b/milvus/tests/compose/Dockerfile index 238dfdedd6d6d..9ec85e75bb888 100644 --- a/milvus/tests/compose/Dockerfile +++ b/milvus/tests/compose/Dockerfile @@ -1,4 +1,3 @@ -# Use the Python slim image as a base FROM python:3.9-slim # Set working directory @@ -8,6 +7,4 @@ WORKDIR /app COPY requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r requirements.txt -# Copy the Python script and healthcheck script COPY script.py /app/script.py -COPY healthcheck_script.sh /app/healthcheck_script.sh diff --git a/milvus/tests/compose/requirements.txt b/milvus/tests/compose/requirements.txt index 1132a5bdcaca1..2cee9519b7f56 100644 --- a/milvus/tests/compose/requirements.txt +++ b/milvus/tests/compose/requirements.txt @@ -1,2 +1,2 @@ -numpy -pymilvus +numpy==1.26.4 +pymilvus==2.2.17 From 2e1d3e8d7e8bdb9ded250b1eacf5bd732e03ab47 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 16 Dec 2024 18:15:46 +0100 Subject: [PATCH 22/31] Switch to deprecated classifier tags for CI --- milvus/manifest.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/milvus/manifest.json b/milvus/manifest.json index 44a7ffd9b74be..c626a0f3d21f9 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -15,9 +15,7 @@ "Supported OS::Linux", "Supported OS::Windows", "Supported OS::macOS", - "Category::AI", - "Category::Data Store", - "Category::Machine Learning & LLM", + "Category::AI/ML", "Offering::Integration", "Submitted Data Type::Metrics", "Submitted Data Type::Logs" From 6ea7d3d5bbb90b594c1e038268ff089f0d617f48 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 16 Dec 2024 18:31:30 +0100 Subject: [PATCH 23/31] Sync labeler --- .github/workflows/config/labeler.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 4fe0c7e782c20..44a07458a0107 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -339,6 +339,8 @@ integration/mesos_slave: - mesos_slave/**/* integration/metabase: - metabase/**/* +integration/milvus: +- milvus/**/* integration/mimecast: - mimecast/**/* integration/mongo: From 8777306e515d2573fcd9cb135b21c09c2e21f345 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 16 Dec 2024 18:40:59 +0100 Subject: [PATCH 24/31] Remove dashboard and add metric check --- milvus/assets/dashboards/milvus_overview.json | 1 - milvus/manifest.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 milvus/assets/dashboards/milvus_overview.json diff --git a/milvus/assets/dashboards/milvus_overview.json b/milvus/assets/dashboards/milvus_overview.json deleted file mode 100644 index e9e23301af626..0000000000000 --- a/milvus/assets/dashboards/milvus_overview.json +++ /dev/null @@ -1 +0,0 @@ -Please build an out-of-the-box dashboard for your integration following our best practices here: https://datadoghq.dev/integrations-core/guidelines/dashboards/#best-practices \ No newline at end of file diff --git a/milvus/manifest.json b/milvus/manifest.json index c626a0f3d21f9..b77db31b7041c 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -34,7 +34,7 @@ }, "metrics": { "prefix": "milvus.", - "check": "", + "check": "milvus.build_info", "metadata_path": "metadata.csv" }, "service_checks": { From caf2c336fe6ff7fbfa7ef917be4ffb0a7fab7538 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 17 Dec 2024 12:16:00 +0100 Subject: [PATCH 25/31] Implement review suggestions --- milvus/datadog_checks/milvus/metrics.py | 17 +- milvus/manifest.json | 2 +- milvus/metadata.csv | 606 ++++++++++++------------ milvus/tests/common.py | 8 +- milvus/tests/test_unit.py | 5 +- 5 files changed, 317 insertions(+), 321 deletions(-) diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py index d3a9a434209f6..bd6a7ff744e59 100644 --- a/milvus/datadog_checks/milvus/metrics.py +++ b/milvus/datadog_checks/milvus/metrics.py @@ -3,12 +3,15 @@ # Licensed under a 3-clause BSD style license (see LICENSE) METRIC_MAP = { - 'milvus_build_info': 'build_info', + 'milvus_build_info': {'type': 'metadata', 'label': 'version', 'name': 'version'}, 'milvus_cgo_active_future_total': 'cgo.active_future_total', 'milvus_cgo_cgo_duration_seconds': 'cgo.cgo_duration_seconds', 'milvus_cgo_cgo_queue_duration_seconds': 'cgo.cgo_queue_duration_seconds', 'milvus_cgo_running_cgo_call_total': 'cgo.running_cgo_call_total', - 'milvus_datacoord_channel_checkpoint_unix_seconds': 'datacoord.channel_checkpoint_unix_seconds', + 'milvus_datacoord_channel_checkpoint_unix_seconds': { + 'name': 'datacoord.time_since_channel_checkpoint', + 'type': 'time_elapsed', + }, 'milvus_datacoord_collection_num': 'datacoord.collection_num', 'milvus_datacoord_consume_datanode_tt_lag_ms': 'datacoord.consume_datanode_tt_lag_ms', 'milvus_datacoord_datanode_num': 'datacoord.datanode_num', @@ -69,7 +72,10 @@ 'milvus_proxy_sync_segment_request_length': 'proxy.sync_segment_request_length', 'milvus_proxy_tt_lag_ms': 'proxy.tt_lag_ms', 'milvus_querycoord_collection_num': 'querycoord.collection_num', - 'milvus_querycoord_current_target_checkpoint_unix_seconds': 'querycoord.current_target_checkpoint_unix_seconds', + 'milvus_querycoord_current_target_checkpoint_unix_seconds': { + 'name': 'querycoord.current_target_checkpoint_unix_seconds', + 'type': 'time_elapsed', + }, 'milvus_querycoord_load_latency': 'querycoord.load.latency', 'milvus_querycoord_load_req_count': 'querycoord.load.req', 'milvus_querycoord_partition_num': 'querycoord.partition_num', @@ -168,8 +174,7 @@ 'go_gc_duration_seconds': 'go.gc_duration_seconds', 'go_goroutines': 'go.goroutines', 'go_info': 'go.info', - 'go_memstats_alloc_bytes': 'go.memstats.alloc_bytes', - # 'go_memstats_alloc_bytes_total': 'go.memstats.alloc_bytes_total', + 'go_memstats_alloc_bytes': {'name': 'go.memstats.alloc_bytes', 'type': 'native_dynamic'}, 'go_memstats_buck_hash_sys_bytes': 'go.memstats.buck_hash_sys_bytes', 'go_memstats_frees': 'go.memstats.frees', 'go_memstats_gc_sys_bytes': 'go.memstats.gc_sys_bytes', @@ -209,7 +214,7 @@ 'process_max_fds': 'process.max_fds', 'process_open_fds': 'process.open_fds', 'process_resident_memory_bytes': 'process.resident_memory_bytes', - 'process_start_time_seconds': 'process.start_time_seconds', + 'process_start_time_seconds': {'name': 'process.start_time_seconds', 'type': 'time_elapsed'}, 'process_virtual_memory_bytes': 'process.virtual_memory.bytes', 'process_virtual_memory_max_bytes': 'process.virtual_memory.max_bytes', 'quant_compute_cnt': 'quant.compute_cnt', diff --git a/milvus/manifest.json b/milvus/manifest.json index b77db31b7041c..bf25c410ae353 100644 --- a/milvus/manifest.json +++ b/milvus/manifest.json @@ -8,7 +8,7 @@ "configuration": "README.md#Setup", "support": "README.md#Support", "changelog": "CHANGELOG.md", - "description": "", + "description": "Monitor the performance and usage of your Milvus deployments.", "title": "Milvus", "media": [], "classifier_tags": [ diff --git a/milvus/metadata.csv b/milvus/metadata.csv index d37fc79da2c67..5187ffe47bf77 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,58 +1,58 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags -milvus.bf_search_cnt.bucket,count,,,,number of bf search per request,0,milvus,,, -milvus.bf_search_cnt.count,count,,,,number of bf search per request,0,milvus,,, -milvus.bf_search_cnt.sum,count,,,,number of bf search per request,0,milvus,,, -milvus.bitset_ratio.bucket,count,,,,bitset ratio,0,milvus,,, -milvus.bitset_ratio.count,count,,,,bitset ratio,0,milvus,,, -milvus.bitset_ratio.sum,count,,,,bitset ratio,0,milvus,,, +milvus.bf_search_cnt.bucket,count,,,,Number of bf search per request,0,milvus,,, +milvus.bf_search_cnt.count,count,,,,Number of bf search per request,0,milvus,,, +milvus.bf_search_cnt.sum,count,,,,Number of bf search per request,0,milvus,,, +milvus.bitset_ratio.bucket,count,,,,Bitset ratio,0,milvus,,, +milvus.bitset_ratio.count,count,,,,Bitset ratio,0,milvus,,, +milvus.bitset_ratio.sum,count,,,,Bitset ratio,0,milvus,,, milvus.build_info,gauge,,,,Build information of milvus,0,milvus,,, -milvus.build_latency.bucket,count,,,,index build latency (s),0,milvus,,, -milvus.build_latency.count,count,,,,index build latency (s),0,milvus,,, -milvus.build_latency.sum,count,,,,index build latency (s),0,milvus,,, -milvus.cache_hit_cnt.bucket,count,,,,cache hit cnt per request,0,milvus,,, -milvus.cache_hit_cnt.count,count,,,,cache hit cnt per request,0,milvus,,, -milvus.cache_hit_cnt.sum,count,,,,cache hit cnt per request,0,milvus,,, +milvus.build_latency.bucket,count,,,,Index build latency (s),0,milvus,,, +milvus.build_latency.count,count,,,,Index build latency (s),0,milvus,,, +milvus.build_latency.sum,count,,second,,Index build latency (s),0,milvus,,, +milvus.cache_hit_cnt.bucket,count,,,,Cache hit cnt per request,0,milvus,,, +milvus.cache_hit_cnt.count,count,,,,Cache hit cnt per request,0,milvus,,, +milvus.cache_hit_cnt.sum,count,,,,Cache hit cnt per request,0,milvus,,, milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,,, -milvus.cgo.cgo_duration_seconds.bucket,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, -milvus.cgo.cgo_duration_seconds.count,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.bucket,count,,,,Histogram of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.count,count,,,,Histogram of cgo call duration in seconds.,0,milvus,,, milvus.cgo.cgo_duration_seconds.sum,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, -milvus.cgo.cgo_queue_duration_seconds.bucket,count,,second,,Duration of cgo call in queue.,0,milvus,,, -milvus.cgo.cgo_queue_duration_seconds.count,count,,second,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.bucket,count,,,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.count,count,,,,Duration of cgo call in queue.,0,milvus,,, milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Duration of cgo call in queue.,0,milvus,,, milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, -milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,channel checkpoint timestamp in unix seconds,0,milvus,,, -milvus.datacoord.collection_num,gauge,,,,number of collections,0,milvus,,, -milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, -milvus.datacoord.datanode_num,gauge,,,,number of data nodes,0,milvus,,, -milvus.datacoord.import_tasks,gauge,,,,the import tasks grouping by type and state,0,milvus,,, -milvus.datacoord.index.node_num,gauge,,,,number of IndexNodes managed by IndexCoord,0,milvus,,, -milvus.datacoord.index.req.count,count,,,,number of building index requests,0,milvus,,, -milvus.datacoord.index.task_count,gauge,,,,number of index tasks of each type,0,milvus,,, -milvus.datacoord.segment_num,gauge,,,,number of segments,0,milvus,,, -milvus.datacoord.stored.binlog_size,gauge,,byte,,binlog size of healthy segments,0,milvus,,, -milvus.datacoord.stored.index_files_size,gauge,,byte,,index files size of the segments,0,milvus,,, -milvus.datacoord.stored.rows_num,gauge,,,,number of stored rows of healthy segment,0,milvus,,, -milvus.datacoord.task_execute_max_latency.bucket,count,,millisecond,,latency of task execute operation,0,milvus,,, -milvus.datacoord.task_execute_max_latency.count,count,,millisecond,,latency of task execute operation,0,milvus,,, -milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,latency of task execute operation,0,milvus,,, -milvus.datacoord.watched_dml_chanel_num,gauge,,,,the num of dml channel watched by datanode,0,milvus,,, -milvus.datanode.autoflush_buffer_op.count,count,,,,count of auto flush buffer operations,0,milvus,,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in unix seconds,0,milvus,,, +milvus.datacoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.datacoord.datanode_num,gauge,,,,Number of data nodes,0,milvus,,, +milvus.datacoord.import_tasks,gauge,,,,The import tasks grouping by type and state,0,milvus,,, +milvus.datacoord.index.node_num,gauge,,,,Number of IndexNodes managed by IndexCoord,0,milvus,,, +milvus.datacoord.index.req.count,count,,,,Number of building index requests,0,milvus,,, +milvus.datacoord.index.task_count,gauge,,,,Number of index tasks of each type,0,milvus,,, +milvus.datacoord.segment_num,gauge,,,,Number of segments,0,milvus,,, +milvus.datacoord.stored.binlog_size,gauge,,byte,,Binlog size of healthy segments,0,milvus,,, +milvus.datacoord.stored.index_files_size,gauge,,byte,,Index files size of the segments,0,milvus,,, +milvus.datacoord.stored.rows_num,gauge,,,,Number of stored rows of healthy segment,0,milvus,,, +milvus.datacoord.task_execute_max_latency.bucket,count,,,,Latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.count,count,,,,Latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,Latency of task execute operation,0,milvus,,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,The num of dml channel watched by datanode,0,milvus,,, +milvus.datanode.autoflush_buffer_op.count,count,,,,Count of auto flush buffer operations,0,milvus,,, milvus.datanode.consume.bytes.count,count,,,,,0,milvus,,, -milvus.datanode.consume.msg.count,count,,,,count of consumed msg,0,milvus,,, -milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, -milvus.datanode.encode_buffer_latency.bucket,count,,millisecond,,latency of encode buffer data,0,milvus,,, -milvus.datanode.encode_buffer_latency.count,count,,millisecond,,latency of encode buffer data,0,milvus,,, -milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,latency of encode buffer data,0,milvus,,, -milvus.datanode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,,, -milvus.datanode.flush.buffer_op.count,count,,,,count of flush buffer operations,0,milvus,,, -milvus.datanode.flush.req.count,count,,,,count of flush request,0,milvus,,, -milvus.datanode.flushed_data.rows.count,count,,,,num of rows flushed to storage,0,milvus,,, -milvus.datanode.flushed_data.size.count,count,,byte,,byte size of data flushed to storage,0,milvus,,, +milvus.datanode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.datanode.encode_buffer_latency.bucket,count,,,,Latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.count,count,,,,Latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,Latency of encode buffer data,0,milvus,,, +milvus.datanode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, +milvus.datanode.flush.buffer_op.count,count,,,,Count of flush buffer operations,0,milvus,,, +milvus.datanode.flush.req.count,count,,,,Count of flush request,0,milvus,,, +milvus.datanode.flushed_data.rows.count,count,,,,Num of rows flushed to storage,0,milvus,,, +milvus.datanode.flushed_data.size.count,count,,byte,,Byte size of data flushed to storage,0,milvus,,, milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, -milvus.datanode.msg.rows.count,count,,,,count of rows consumed from msgStream,0,milvus,,, -milvus.datanode.save_latency.bucket,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, -milvus.datanode.save_latency.count,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, -milvus.datanode.save_latency.sum,count,,millisecond,,latency of saving flush data to storage,0,milvus,,, +milvus.datanode.msg.rows.count,count,,,,Count of rows consumed from msgStream,0,milvus,,, +milvus.datanode.save_latency.bucket,count,,,,Latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.count,count,,,,Latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.sum,count,,millisecond,,Latency of saving flush data to storage,0,milvus,,, milvus.diskann.range_search_iters.bucket,count,,,,DISKANN range search iterations,0,milvus,,, milvus.diskann.range_search_iters.count,count,,,,DISKANN range search iterations,0,milvus,,, milvus.diskann.range_search_iters.sum,count,,,,DISKANN range search iterations,0,milvus,,, @@ -62,27 +62,27 @@ milvus.diskann.search_hops.sum,count,,,,DISKANN search hops,0,milvus,,, milvus.diskann_bitset_ratio.bucket,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, milvus.diskann_bitset_ratio.count,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, milvus.diskann_bitset_ratio.sum,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, -milvus.exec_latency.bucket,count,,,,execute latency per request,0,milvus,,, -milvus.exec_latency.count,count,,,,execute latency per request,0,milvus,,, -milvus.exec_latency.sum,count,,,,execute latency per request,0,milvus,,, -milvus.filter.connectivity_ratio.bucket,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.connectivity_ratio.count,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.connectivity_ratio.sum,count,,,,avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.bucket,count,,,,avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.count,count,,,,avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.sum,count,,,,avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.bucket,count,,,,mv change base cnt per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.count,count,,,,mv change base cnt per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.sum,count,,,,mv change base cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.bucket,count,,,,mv only cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.count,count,,,,mv only cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.sum,count,,,,mv only cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.flushed_segment_file_num.bucket,count,,,,the num of files for flushed segment,0,milvus,,, -milvus.flushed_segment_file_num.count,count,,,,the num of files for flushed segment,0,milvus,,, -milvus.flushed_segment_file_num.sum,count,,,,the num of files for flushed segment,0,milvus,,, +milvus.exec_latency.bucket,count,,,,Execute latency per request,0,milvus,,, +milvus.exec_latency.count,count,,,,Execute latency per request,0,milvus,,, +milvus.exec_latency.sum,count,,,,Execute latency per request,0,milvus,,, +milvus.filter.connectivity_ratio.bucket,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.count,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.sum,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.bucket,count,,,,Avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.count,count,,,,Avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.sum,count,,,,Avg mv activated fields per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.bucket,count,,,,Mv change base cnt per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.count,count,,,,Mv change base cnt per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.sum,count,,,,Mv change base cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.bucket,count,,,,Mv only cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.count,count,,,,Mv only cnt per request,0,milvus,,, +milvus.filter.mv.only_cnt.sum,count,,,,Mv only cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, +milvus.flushed_segment_file_num.bucket,count,,,,The num of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.count,count,,,,The num of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.sum,count,,,,The num of files for flushed segment,0,milvus,,, milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, @@ -112,31 +112,31 @@ milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stac milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,,, milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,,, milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,,, -milvus.graph_search_cnt.bucket,count,,,,number of graph search per request,0,milvus,,, -milvus.graph_search_cnt.count,count,,,,number of graph search per request,0,milvus,,, -milvus.graph_search_cnt.sum,count,,,,number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.bucket,count,,,,Number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.count,count,,,,Number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.sum,count,,,,Number of graph search per request,0,milvus,,, milvus.hnsw.bitset_ratio.bucket,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, milvus.hnsw.bitset_ratio.count,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, milvus.hnsw.bitset_ratio.sum,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, milvus.hnsw.search_hops.bucket,count,,,,HNSW search hops in layer 0,0,milvus,,, milvus.hnsw.search_hops.count,count,,,,HNSW search hops in layer 0,0,milvus,,, milvus.hnsw.search_hops.sum,count,,,,HNSW search hops in layer 0,0,milvus,,, -milvus.indexnode.build_index_latency.bucket,count,,millisecond,,latency of build index for segment,0,milvus,,, -milvus.indexnode.build_index_latency.count,count,,millisecond,,latency of build index for segment,0,milvus,,, -milvus.indexnode.build_index_latency.sum,count,,millisecond,,latency of build index for segment,0,milvus,,, -milvus.indexnode.encode_index_latency.bucket,count,,millisecond,,latency of encoding the index file,0,milvus,,, -milvus.indexnode.encode_index_latency.count,count,,millisecond,,latency of encoding the index file,0,milvus,,, -milvus.indexnode.encode_index_latency.sum,count,,millisecond,,latency of encoding the index file,0,milvus,,, -milvus.indexnode.index.task.count,count,,,,number of tasks that index node received,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.bucket,count,,millisecond,,latency of index task in queue,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.count,count,,millisecond,,latency of index task in queue,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,latency of index task in queue,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.bucket,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.count,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.save_index_latency.bucket,count,,millisecond,,latency of saving the index file,0,milvus,,, -milvus.indexnode.save_index_latency.count,count,,millisecond,,latency of saving the index file,0,milvus,,, -milvus.indexnode.save_index_latency.sum,count,,millisecond,,latency of saving the index file,0,milvus,,, +milvus.indexnode.build_index_latency.bucket,count,,,,Latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.count,count,,,,Latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.sum,count,,millisecond,,Latency of build index for segment,0,milvus,,, +milvus.indexnode.encode_index_latency.bucket,count,,,,Latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.count,count,,,,Latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.sum,count,,millisecond,,Latency of encoding the index file,0,milvus,,, +milvus.indexnode.index.task.count,count,,,,Number of tasks that index node received,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.bucket,count,,,,Latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.count,count,,,,Latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,Latency of index task in queue,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.bucket,count,,,,Latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.count,count,,,,Latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,Latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.save_index_latency.bucket,count,,,,Latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.count,count,,,,Latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.sum,count,,millisecond,,Latency of saving the index file,0,milvus,,, milvus.internal.core_search_latency.bucket,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, milvus.internal.core_search_latency.count,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, milvus.internal.core_search_latency.sum,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, @@ -153,29 +153,29 @@ milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segmen milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,,, milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, -milvus.internal.storage.request_latency.sum,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, -milvus.io_cnt.bucket,count,,,,io cnt per request,0,milvus,,, -milvus.io_cnt.count,count,,,,io cnt per request,0,milvus,,, -milvus.io_cnt.sum,count,,,,io cnt per request,0,milvus,,, -milvus.ivf_search_cnt.bucket,count,,,,number of ivf search per request,0,milvus,,, -milvus.ivf_search_cnt.count,count,,,,number of ivf search per request,0,milvus,,, -milvus.ivf_search_cnt.sum,count,,,,number of ivf search per request,0,milvus,,, -milvus.load_latency.bucket,count,,,,index load latency (ms),0,milvus,,, -milvus.load_latency.count,count,,,,index load latency (ms),0,milvus,,, -milvus.load_latency.sum,count,,,,index load latency (ms),0,milvus,,, -milvus.meta.kv_size.bucket,count,,,,kv size stats,0,milvus,,, -milvus.meta.kv_size.count,count,,,,kv size stats,0,milvus,,, -milvus.meta.kv_size.sum,count,,,,kv size stats,0,milvus,,, -milvus.meta.op.count,count,,,,count of meta operation,0,milvus,,, -milvus.meta.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.meta.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.meta.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.msg_queue_consumer_num,gauge,,,,number of consumers,0,milvus,,, -milvus.msgstream.op.count,count,,,,count of stream message operation,0,milvus,,, -milvus.msgstream.request_latency.bucket,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.msgstream.request_latency.count,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.msgstream.request_latency.sum,count,,millisecond,,request latency on the client side,0,milvus,,, -milvus.num_node,gauge,,,,number of nodes and coordinates,0,milvus,,, +milvus.internal.storage.request_latency.sum,count,,millisecond,,[cpp]request latency(ms) on the client side,0,milvus,,, +milvus.io_cnt.bucket,count,,,,Io cnt per request,0,milvus,,, +milvus.io_cnt.count,count,,,,Io cnt per request,0,milvus,,, +milvus.io_cnt.sum,count,,,,Io cnt per request,0,milvus,,, +milvus.ivf_search_cnt.bucket,count,,,,Number of ivf search per request,0,milvus,,, +milvus.ivf_search_cnt.count,count,,,,Number of ivf search per request,0,milvus,,, +milvus.ivf_search_cnt.sum,count,,,,Number of ivf search per request,0,milvus,,, +milvus.load_latency.bucket,count,,,,Index load latency (ms),0,milvus,,, +milvus.load_latency.count,count,,,,Index load latency (ms),0,milvus,,, +milvus.load_latency.sum,count,,millisecond,,Index load latency (ms),0,milvus,,, +milvus.meta.kv_size.bucket,count,,,,Kv size stats,0,milvus,,, +milvus.meta.kv_size.count,count,,,,Kv size stats,0,milvus,,, +milvus.meta.kv_size.sum,count,,,,Kv size stats,0,milvus,,, +milvus.meta.op.count,count,,,,Count of meta operation,0,milvus,,, +milvus.meta.request_latency.bucket,count,,,,Request latency on the client side,0,milvus,,, +milvus.meta.request_latency.count,count,,,,Request latency on the client side,0,milvus,,, +milvus.meta.request_latency.sum,count,,millisecond,,Request latency on the client side,0,milvus,,, +milvus.msg_queue_consumer_num,gauge,,,,Number of consumers,0,milvus,,, +milvus.msgstream.op.count,count,,,,Count of stream message operation,0,milvus,,, +milvus.msgstream.request_latency.bucket,count,,,,Request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.count,count,,,,Request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.sum,count,,millisecond,,Request latency on the client side,0,milvus,,, +milvus.num_node,gauge,,,,Number of nodes and coordinates,0,milvus,,, milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,,, milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,,, milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,,, @@ -183,205 +183,205 @@ milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,m milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,,, milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,,, milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,,, -milvus.proxy.apply.pk_latency.bucket,count,,millisecond,,latency that apply primary key,0,milvus,,, -milvus.proxy.apply.pk_latency.count,count,,millisecond,,latency that apply primary key,0,milvus,,, -milvus.proxy.apply.pk_latency.sum,count,,millisecond,,latency that apply primary key,0,milvus,,, -milvus.proxy.apply.timestamp_latency.bucket,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.apply.timestamp_latency.count,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.assign_segmentID_latency.bucket,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, -milvus.proxy.assign_segmentID_latency.count,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, -milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.apply.pk_latency.bucket,count,,,,Latency that apply primary key,0,milvus,,, +milvus.proxy.apply.pk_latency.count,count,,,,Latency that apply primary key,0,milvus,,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,Latency that apply primary key,0,milvus,,, +milvus.proxy.apply.timestamp_latency.bucket,count,,,,Latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.apply.timestamp_latency.count,count,,,,Latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,Latency that proxy apply timestamp,0,milvus,,, +milvus.proxy.assign_segmentID_latency.bucket,count,,,,Latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.assign_segmentID_latency.count,count,,,,Latency that proxy get segmentID from dataCoord,0,milvus,,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,Latency that proxy get segmentID from dataCoord,0,milvus,,, milvus.proxy.cache.hit.count,count,,,,count of cache hits/miss,0,milvus,,, -milvus.proxy.cache.update_latency.bucket,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, -milvus.proxy.cache.update_latency.count,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, -milvus.proxy.cache.update_latency.sum,count,,millisecond,,latency that proxy update cache when cache miss,0,milvus,,, -milvus.proxy.delete_vectors.count,count,,,,counter of vectors successfully deleted,0,milvus,,, -milvus.proxy.msgstream_obj_num,gauge,,,,number of MsgStream objects per physical channel,0,milvus,,, -milvus.proxy.mutation_send_latency.bucket,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, -milvus.proxy.mutation_send_latency.count,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, -milvus.proxy.mutation_send_latency.sum,count,,millisecond,,latency that proxy send insert request to MsgStream,0,milvus,,, -milvus.proxy.rate_limit_req.count,count,,,,count of operation executed,0,milvus,,, -milvus.proxy.report_value.count,count,,,,report value about the request,0,milvus,,, -milvus.proxy.req.count,count,,,,count of operation executed,0,milvus,,, -milvus.proxy.req.in_queue_latency.bucket,count,,millisecond,,latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.in_queue_latency.count,count,,millisecond,,latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.latency.bucket,count,,millisecond,,latency of each request,0,milvus,,, -milvus.proxy.req.latency.count,count,,millisecond,,latency of each request,0,milvus,,, -milvus.proxy.req.latency.sum,count,,millisecond,,latency of each request,0,milvus,,, -milvus.proxy.send_bytes.count,count,,,,count of bytes sent back to sdk,0,milvus,,, -milvus.proxy.sq.decode_result_latency.bucket,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.decode_result_latency.count,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.bucket,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.count,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.bucket,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.count,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sync_segment_request_length.bucket,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.sync_segment_request_length.count,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.sync_segment_request_length.sum,count,,,,the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, -milvus.quant.compute_cnt.bucket,count,,,,quant compute cnt per request,0,milvus,,, -milvus.quant.compute_cnt.count,count,,,,quant compute cnt per request,0,milvus,,, -milvus.quant.compute_cnt.sum,count,,,,quant compute cnt per request,0,milvus,,, -milvus.querycoord.collection_num,gauge,,,,number of collections,0,milvus,,, -milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,current target checkpoint timestamp in unix seconds,0,milvus,,, -milvus.querycoord.load.latency.bucket,count,,millisecond,,latency of load the entire collection,0,milvus,,, -milvus.querycoord.load.latency.count,count,,millisecond,,latency of load the entire collection,0,milvus,,, -milvus.querycoord.load.latency.sum,count,,millisecond,,latency of load the entire collection,0,milvus,,, -milvus.querycoord.load.req.count,count,,,,count of load request,0,milvus,,, -milvus.querycoord.partition_num,gauge,,,,number of partitions,0,milvus,,, -milvus.querycoord.querynode_num,gauge,,,,number of QueryNodes managered by QueryCoord,0,milvus,,, -milvus.querycoord.release.latency.bucket,count,,millisecond,,latency of release request,0,milvus,,, -milvus.querycoord.release.latency.count,count,,millisecond,,latency of release request,0,milvus,,, -milvus.querycoord.release.latency.sum,count,,millisecond,,latency of release request,0,milvus,,, -milvus.querycoord.release.req.count,count,,,,count of release request,0,milvus,,, +milvus.proxy.cache.update_latency.bucket,count,,,,Latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.cache.update_latency.count,count,,,,Latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,Latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.delete_vectors.count,count,,,,Counter of vectors successfully deleted,0,milvus,,, +milvus.proxy.msgstream_obj_num,gauge,,,,Number of MsgStream objects per physical channel,0,milvus,,, +milvus.proxy.mutation_send_latency.bucket,count,,,,Latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.mutation_send_latency.count,count,,,,Latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,Latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.rate_limit_req.count,count,,,,Count of operation executed,0,milvus,,, +milvus.proxy.report_value.count,count,,,,Report value about the request,0,milvus,,, +milvus.proxy.req.count,count,,,,Count of operation executed,0,milvus,,, +milvus.proxy.req.in_queue_latency.bucket,count,,,,Latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.count,count,,,,Latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,Latency which request waits in the queue,0,milvus,,, +milvus.proxy.req.latency.bucket,count,,,,Latency of each request,0,milvus,,, +milvus.proxy.req.latency.count,count,,,,Latency of each request,0,milvus,,, +milvus.proxy.req.latency.sum,count,,millisecond,,Latency of each request,0,milvus,,, +milvus.proxy.send_bytes.count,count,,,,Count of bytes sent back to sdk,0,milvus,,, +milvus.proxy.sq.decode_result_latency.bucket,count,,,,Latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.decode_result_latency.count,count,,,,Latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,Latency that proxy decodes the search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,,,Latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.count,count,,,,Latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,Latency that proxy reduces search result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.bucket,count,,,,Latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.count,count,,,,Latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,Latency that proxy waits for the result,0,milvus,,, +milvus.proxy.sync_segment_request_length.bucket,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.count,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.sum,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.quant.compute_cnt.bucket,count,,,,Quant compute cnt per request,0,milvus,,, +milvus.quant.compute_cnt.count,count,,,,Quant compute cnt per request,0,milvus,,, +milvus.quant.compute_cnt.sum,count,,,,Quant compute cnt per request,0,milvus,,, +milvus.querycoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,Current target checkpoint timestamp in unix seconds,0,milvus,,, +milvus.querycoord.load.latency.bucket,count,,,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.count,count,,,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.sum,count,,millisecond,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.req.count,count,,,,Count of load request,0,milvus,,, +milvus.querycoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, +milvus.querycoord.querynode_num,gauge,,,,Number of QueryNodes managered by QueryCoord,0,milvus,,, +milvus.querycoord.release.latency.bucket,count,,,,Latency of release request,0,milvus,,, +milvus.querycoord.release.latency.count,count,,,,Latency of release request,0,milvus,,, +milvus.querycoord.release.latency.sum,count,,millisecond,,Latency of release request,0,milvus,,, +milvus.querycoord.release.req.count,count,,,,Count of release request,0,milvus,,, milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,,, -milvus.querynode.apply_bf_latency.bucket,count,,millisecond,,apply bf cost in ms,0,milvus,,, -milvus.querynode.apply_bf_latency.count,count,,millisecond,,apply bf cost in ms,0,milvus,,, -milvus.querynode.apply_bf_latency.sum,count,,millisecond,,apply bf cost in ms,0,milvus,,, -milvus.querynode.collection_num,gauge,,,,number of collections loaded,0,milvus,,, +milvus.querynode.apply_bf_latency.bucket,count,,,,Apply bf cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.count,count,,,,Apply bf cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.sum,count,,millisecond,,Apply bf cost in ms,0,milvus,,, +milvus.querynode.collection_num,gauge,,,,Number of collections loaded,0,milvus,,, milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,,, -milvus.querynode.consume.msg.count,count,,,,count of consumed msg,0,milvus,,, -milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, -milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,number of bytes evicted from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.count,count,,,,number of segments evicted from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.duration.count,count,,millisecond,,total time cost of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.bucket,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.count,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,global duration of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.bytes.count,count,,byte,,number of bytes loaded from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.count,count,,,,number of segments loaded from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.duration.count,count,,millisecond,,total time cost of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.bucket,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.count,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,global duration of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.used_size,gauge,,mebibyte,,disk used size(MB),0,milvus,,, -milvus.querynode.dml_vchannel_num,gauge,,,,number of dmlChannels watched,0,milvus,,, -milvus.querynode.entity.num,gauge,,,,"number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, -milvus.querynode.entity.size,gauge,,byte,,"entities' memory size, clustered by collection and state",0,milvus,,, +milvus.querynode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,Number of bytes evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.count,count,,,,Number of segments evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.duration.count,count,,,,Total time cost of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.bucket,count,,,,Global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.count,count,,,,Global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,Global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.bytes.count,count,,byte,,Number of bytes loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.count,count,,,,Number of segments loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.duration.count,count,,,,Total time cost of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.bucket,count,,,,Global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.count,count,,,,Global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,Global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.used_size,gauge,,mebibyte,,Disk used size(MB),0,milvus,,, +milvus.querynode.dml_vchannel_num,gauge,,,,Number of dmlChannels watched,0,milvus,,, +milvus.querynode.entity.num,gauge,,,,"Number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, +milvus.querynode.entity.size,gauge,,byte,,"Entities' memory size, clustered by collection and state",0,milvus,,, milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,,, -milvus.querynode.flowgraph_num,gauge,,,,number of flowgraphs,0,milvus,,, -milvus.querynode.forward_delete_latency.bucket,count,,millisecond,,forward delete cost in ms,0,milvus,,, -milvus.querynode.forward_delete_latency.count,count,,millisecond,,forward delete cost in ms,0,milvus,,, -milvus.querynode.forward_delete_latency.sum,count,,millisecond,,forward delete cost in ms,0,milvus,,, -milvus.querynode.load.index_latency.bucket,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, -milvus.querynode.load.index_latency.count,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, -milvus.querynode.load.index_latency.sum,count,,millisecond,,"latency of load per segment's index, in milliseconds",0,milvus,,, -milvus.querynode.load.segment.concurrency,gauge,,,,number of concurrent loading segments in QueryNode,0,milvus,,, -milvus.querynode.load.segment.latency.bucket,count,,millisecond,,latency of load per segment,0,milvus,,, -milvus.querynode.load.segment.latency.count,count,,millisecond,,latency of load per segment,0,milvus,,, -milvus.querynode.load.segment.latency.sum,count,,millisecond,,latency of load per segment,0,milvus,,, +milvus.querynode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, +milvus.querynode.forward_delete_latency.bucket,count,,,,Forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.count,count,,,,Forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.sum,count,,millisecond,,Forward delete cost in ms,0,milvus,,, +milvus.querynode.load.index_latency.bucket,count,,,,"Latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.count,count,,,,"Latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.sum,count,,millisecond,,"Latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.segment.concurrency,gauge,,,,Number of concurrent loading segments in QueryNode,0,milvus,,, +milvus.querynode.load.segment.latency.bucket,count,,,,Latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.count,count,,,,Latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.sum,count,,millisecond,,Latency of load per segment,0,milvus,,, milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, -milvus.querynode.partition_num,gauge,,,,number of partitions loaded,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.bucket,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.count,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,process insert or delete cost in ms,0,milvus,,, -milvus.querynode.read_task.concurrency,gauge,,,,number of concurrent executing read tasks in QueryNode,0,milvus,,, -milvus.querynode.read_task.ready_len,gauge,,,,number of ready read tasks in readyQueue,0,milvus,,, -milvus.querynode.read_task.unsolved_len,gauge,,,,number of unsolved read tasks in unsolvedQueue,0,milvus,,, -milvus.querynode.search.group.nq.bucket,count,,,,the number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.nq.count,count,,,,the number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.nq.sum,count,,,,the number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.bucket,count,,,,the number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.count,count,,,,the number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.sum,count,,,,the number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.bucket,count,,,,the topK of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.count,count,,,,the topK of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.sum,count,,,,the topK of each grouped search task,0,milvus,,, -milvus.querynode.search.nq.bucket,count,,,,the number of queries of each search task,0,milvus,,, -milvus.querynode.search.nq.count,count,,,,the number of queries of each search task,0,milvus,,, -milvus.querynode.search.nq.sum,count,,,,the number of queries of each search task,0,milvus,,, -milvus.querynode.search.topk.bucket,count,,,,the top of each search task,0,milvus,,, -milvus.querynode.search.topk.count,count,,,,the top of each search task,0,milvus,,, -milvus.querynode.search.topk.sum,count,,,,the top of each search task,0,milvus,,, -milvus.querynode.segment.access.count,count,,,,number of segments accessed,0,milvus,,, -milvus.querynode.segment.access.duration.count,count,,millisecond,,total time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.bucket,count,,millisecond,,global time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.count,count,,millisecond,,global time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,global time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.wait_cache.count,count,,,,number of segments waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.duration.count,count,,millisecond,,total time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.count,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.latency_per_vector.bucket,count,,millisecond,,one vector's search latency per segment,0,milvus,,, -milvus.querynode.segment.latency_per_vector.count,count,,millisecond,,one vector's search latency per segment,0,milvus,,, -milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,one vector's search latency per segment,0,milvus,,, -milvus.querynode.segment.num,gauge,,,,"number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,,, -milvus.querynode.sq.core_latency.bucket,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.core_latency.count,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.core_latency.sum,count,,millisecond,,latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.queue.latency.bucket,count,,millisecond,,latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.latency.count,count,,millisecond,,latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.latency.sum,count,,millisecond,,latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.bucket,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.count,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.reduce_latency.bucket,count,,millisecond,,latency of reduce search or query result,0,milvus,,, -milvus.querynode.sq.reduce_latency.count,count,,millisecond,,latency of reduce search or query result,0,milvus,,, -milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,latency of reduce search or query result,0,milvus,,, +milvus.querynode.partition_num,gauge,,,,Number of partitions loaded,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.bucket,count,,,,Process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.count,count,,,,Process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,Process insert or delete cost in ms,0,milvus,,, +milvus.querynode.read_task.concurrency,gauge,,,,Number of concurrent executing read tasks in QueryNode,0,milvus,,, +milvus.querynode.read_task.ready_len,gauge,,,,Number of ready read tasks in readyQueue,0,milvus,,, +milvus.querynode.read_task.unsolved_len,gauge,,,,Number of unsolved read tasks in unsolvedQueue,0,milvus,,, +milvus.querynode.search.group.nq.bucket,count,,,,The number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.count,count,,,,The number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.sum,count,,,,The number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.bucket,count,,,,The number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.count,count,,,,The number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.sum,count,,,,The number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.bucket,count,,,,The topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.count,count,,,,The topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.sum,count,,,,The topK of each grouped search task,0,milvus,,, +milvus.querynode.search.nq.bucket,count,,,,The number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.count,count,,,,The number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.sum,count,,,,The number of queries of each search task,0,milvus,,, +milvus.querynode.search.topk.bucket,count,,,,The top of each search task,0,milvus,,, +milvus.querynode.search.topk.count,count,,,,The top of each search task,0,milvus,,, +milvus.querynode.search.topk.sum,count,,,,The top of each search task,0,milvus,,, +milvus.querynode.segment.access.count,count,,,,Number of segments accessed,0,milvus,,, +milvus.querynode.segment.access.duration.count,count,,,,Total time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.bucket,count,,,,Global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.count,count,,,,Global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,Global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.wait_cache.count,count,,,,Number of segments waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.duration.count,count,,,,Total time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,,,Global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.count,count,,,,Global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,Global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.latency_per_vector.bucket,count,,,,One vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.count,count,,,,One vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,One vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.num,gauge,,,,"Number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,,, +milvus.querynode.sq.core_latency.bucket,count,,,,Latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.count,count,,,,Latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.sum,count,,millisecond,,Latency of search or query latency in segcore,0,milvus,,, +milvus.querynode.sq.queue.latency.bucket,count,,,,Latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.count,count,,,,Latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.sum,count,,millisecond,,Latency of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.bucket,count,,,,Latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.count,count,,,,Latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,Latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.reduce_latency.bucket,count,,,,Latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.count,count,,,,Latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,Latency of reduce search or query result,0,milvus,,, milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,,, -milvus.querynode.sq.req.latency.bucket,count,,millisecond,,latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.req.latency.count,count,,millisecond,,latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.req.latency.sum,count,,millisecond,,latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.segment_latency.bucket,count,,millisecond,,latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.segment_latency.count,count,,millisecond,,latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.segment_latency.sum,count,,millisecond,,latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.bucket,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.count,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,latency of search or query to wait for tsafe,0,milvus,,, -milvus.querynode.wait_processing_msg,gauge,,,,count of wait processing msg,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.bucket,count,,millisecond,,latency of watch dml channel,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.count,count,,millisecond,,latency of watch dml channel,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,latency of watch dml channel,0,milvus,,, -milvus.queue.latency.bucket,count,,,,queue latency per request,0,milvus,,, -milvus.queue.latency.count,count,,,,queue latency per request,0,milvus,,, -milvus.queue.latency.sum,count,,,,queue latency per request,0,milvus,,, -milvus.range_search_latency.bucket,count,,,,range search latency (ms),0,milvus,,, -milvus.range_search_latency.count,count,,,,range search latency (ms),0,milvus,,, -milvus.range_search_latency.sum,count,,,,range search latency (ms),0,milvus,,, -milvus.raw_compute_cnt.bucket,count,,,,raw compute cnt per request,0,milvus,,, -milvus.raw_compute_cnt.count,count,,,,raw compute cnt per request,0,milvus,,, -milvus.raw_compute_cnt.sum,count,,,,raw compute cnt per request,0,milvus,,, -milvus.re_search_cnt.bucket,count,,,,number of fallback search per request,0,milvus,,, -milvus.re_search_cnt.count,count,,,,number of fallback search per request,0,milvus,,, -milvus.re_search_cnt.sum,count,,,,number of fallback search per request,0,milvus,,, -milvus.rootcoord.collection_num,gauge,,,,number of collections,0,milvus,,, -milvus.rootcoord.credential_num,gauge,,,,number of credentials,0,milvus,,, -milvus.rootcoord.ddl_req.count,count,,,,count of DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.bucket,count,,millisecond,,latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.count,count,,millisecond,,latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.count,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,latency of each DDL operations in queue,0,milvus,,, -milvus.rootcoord.dml_channel_num,gauge,,,,number of DML channels,0,milvus,,, -milvus.rootcoord.entity_num,gauge,,,,"number of entities, clustered by collection and their status(loaded/total)",0,milvus,,, +milvus.querynode.sq.req.latency.bucket,count,,,,Latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.count,count,,,,Latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.sum,count,,millisecond,,Latency of Search or query requests,0,milvus,,, +milvus.querynode.sq.segment_latency.bucket,count,,,,Latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.count,count,,,,Latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.sum,count,,millisecond,,Latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.bucket,count,,,,Latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.count,count,,,,Latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,Latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.wait_processing_msg,gauge,,,,Count of wait processing msg,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Latency of watch dml channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.count,count,,,,Latency of watch dml channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Latency of watch dml channel,0,milvus,,, +milvus.queue.latency.bucket,count,,,,Queue latency per request,0,milvus,,, +milvus.queue.latency.count,count,,,,Queue latency per request,0,milvus,,, +milvus.queue.latency.sum,count,,,,Queue latency per request,0,milvus,,, +milvus.range_search_latency.bucket,count,,,,Range search latency (ms),0,milvus,,, +milvus.range_search_latency.count,count,,,,Range search latency (ms),0,milvus,,, +milvus.range_search_latency.sum,count,,millisecond,,Range search latency (ms),0,milvus,,, +milvus.raw_compute_cnt.bucket,count,,,,Raw compute cnt per request,0,milvus,,, +milvus.raw_compute_cnt.count,count,,,,Raw compute cnt per request,0,milvus,,, +milvus.raw_compute_cnt.sum,count,,,,Raw compute cnt per request,0,milvus,,, +milvus.re_search_cnt.bucket,count,,,,Number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.count,count,,,,Number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.sum,count,,,,Number of fallback search per request,0,milvus,,, +milvus.rootcoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.rootcoord.credential_num,gauge,,,,Number of credentials,0,milvus,,, +milvus.rootcoord.ddl_req.count,count,,,,Count of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.bucket,count,,,,Latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.count,count,,,,Latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,Latency of each DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,,,Latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.count,count,,,,Latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,Latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.dml_channel_num,gauge,,,,Number of DML channels,0,milvus,,, +milvus.rootcoord.entity_num,gauge,,,,"Number of entities, clustered by collection and their status(loaded/total)",0,milvus,,, milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,,, -milvus.rootcoord.id_alloc.count,count,,,,count of ID allocated,0,milvus,,, -milvus.rootcoord.indexed_entity_num,gauge,,,,"indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,,, -milvus.rootcoord.msgstream_obj_num,gauge,,,,number of message streams,0,milvus,,, +milvus.rootcoord.id_alloc.count,count,,,,Count of ID allocated,0,milvus,,, +milvus.rootcoord.indexed_entity_num,gauge,,,,"Indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,,, +milvus.rootcoord.msgstream_obj_num,gauge,,,,Number of message streams,0,milvus,,, milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,,, -milvus.rootcoord.partition_num,gauge,,,,number of partitions,0,milvus,,, -milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,now time minus tt per physical channel,0,milvus,,, -milvus.rootcoord.proxy_num,gauge,,,,number of proxy nodes managered by rootcoord,0,milvus,,, -milvus.rootcoord.qn_mem_high_water_level,gauge,,,,querynode memory high water level,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.bucket,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.count,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,latency of synchronizing timetick message,0,milvus,,, -milvus.rootcoord.timestamp,gauge,,,,latest timestamp allocated in memory,0,milvus,,, -milvus.rootcoord.timestamp_saved,gauge,,,,timestamp saved in meta storage,0,milvus,,, +milvus.rootcoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.rootcoord.proxy_num,gauge,,,,Number of proxy nodes managered by rootcoord,0,milvus,,, +milvus.rootcoord.qn_mem_high_water_level,gauge,,,,Querynode memory high water level,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.bucket,count,,,,Latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.count,count,,,,Latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,Latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.timestamp,gauge,,,,Latest timestamp allocated in memory,0,milvus,,, +milvus.rootcoord.timestamp_saved,gauge,,,,Timestamp saved in meta storage,0,milvus,,, milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,,, -milvus.search.latency.bucket,count,,,,search latency (ms),0,milvus,,, -milvus.search.latency.count,count,,,,search latency (ms),0,milvus,,, -milvus.search.latency.sum,count,,,,search latency (ms),0,milvus,,, -milvus.search.topk.bucket,count,,,,search topk,0,milvus,,, -milvus.search.topk.count,count,,,,search topk,0,milvus,,, -milvus.search.topk.sum,count,,,,search topk,0,milvus,,, -milvus.storage.kv_size,gauge,,,,kv size stats,0,milvus,,, -milvus.storage.op.count,count,,,,count of persistent data operation,0,milvus,,, -milvus.storage.request_latency,gauge,,,,request latency on the client side,0,milvus,,, +milvus.search.latency.bucket,count,,,,Search latency (ms),0,milvus,,, +milvus.search.latency.count,count,,,,Search latency (ms),0,milvus,,, +milvus.search.latency.sum,count,,millisecond,,Search latency (ms),0,milvus,,, +milvus.search.topk.bucket,count,,,,Search topk,0,milvus,,, +milvus.search.topk.count,count,,,,Search topk,0,milvus,,, +milvus.search.topk.sum,count,,,,Search topk,0,milvus,,, +milvus.storage.kv_size,gauge,,,,Kv size stats,0,milvus,,, +milvus.storage.op.count,count,,,,Count of persistent data operation,0,milvus,,, +milvus.storage.request_latency,gauge,,,,Request latency on the client side,0,milvus,,, diff --git a/milvus/tests/common.py b/milvus/tests/common.py index 7bf28a3e887e7..4b2999f41bee7 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -28,7 +28,7 @@ def get_fixture_path(filename): 'milvus.go.goroutines': 'gauge', 'milvus.go.info': 'gauge', 'milvus.go.memstats.alloc_bytes': 'gauge', - # 'milvus.go.memstats.alloc_bytes.count': 'monotonic_count', + 'milvus.go.memstats.alloc_bytes.count': 'monotonic_count', 'milvus.go.memstats.buck_hash_sys_bytes': 'gauge', 'milvus.go.memstats.frees.count': 'monotonic_count', 'milvus.go.memstats.gc_sys_bytes': 'gauge', @@ -51,7 +51,6 @@ def get_fixture_path(filename): 'milvus.go.memstats.stack.sys_bytes': 'gauge', 'milvus.go.memstats.sys_bytes': 'gauge', 'milvus.go.threads': 'gauge', - 'milvus.build_info': 'gauge', 'milvus.cgo.active_future_total': 'gauge', 'milvus.cgo.cgo_duration_seconds.sum': 'monotonic_count', 'milvus.cgo.cgo_duration_seconds.count': 'monotonic_count', @@ -415,8 +414,3 @@ def get_fixture_path(filename): 'milvus.storage.op_count': 'count', 'milvus.storage.request_latency': 'gauge', } - - -RENAMED_LABELS = { - "milvus.build_info": 'milvus_version:v2.4.15', -} diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py index 23af6f5d0c53a..cbf2e608c731a 100644 --- a/milvus/tests/test_unit.py +++ b/milvus/tests/test_unit.py @@ -7,7 +7,7 @@ from datadog_checks.dev.utils import get_metadata_metrics from datadog_checks.milvus import MilvusCheck -from .common import RENAMED_LABELS, STANDALONE_TEST_METRICS, get_fixture_path +from .common import STANDALONE_TEST_METRICS, get_fixture_path def test_check(dd_run_check, aggregator, instance, mock_http_response): @@ -20,9 +20,6 @@ def test_check(dd_run_check, aggregator, instance, mock_http_response): aggregator.assert_metric(metric, metric_type=aggregator.METRIC_ENUM_MAP[metric_type]) aggregator.assert_metric_has_tag(metric, 'test:tag') - for metric, tag in RENAMED_LABELS.items(): - aggregator.assert_metric_has_tag(metric, tag) - aggregator.assert_all_metrics_covered() aggregator.assert_metrics_using_metadata(get_metadata_metrics()) From 8ee167008091861d43f14cb14bb7ada4c31fd23c Mon Sep 17 00:00:00 2001 From: David Kirov Date: Thu, 19 Dec 2024 15:07:15 +0100 Subject: [PATCH 26/31] Incorporate documentation suggestions --- milvus/README.md | 13 +- milvus/metadata.csv | 506 ++++++++++++++++++++++---------------------- 2 files changed, 256 insertions(+), 263 deletions(-) diff --git a/milvus/README.md b/milvus/README.md index 1a951270ac9ad..85f293f5c2cfb 100644 --- a/milvus/README.md +++ b/milvus/README.md @@ -2,12 +2,7 @@ ## Overview -This check monitors [Milvus][1] through the Datadog Agent. - -Include a high level overview of what this integration does: -- What does your product do (in 1-2 sentences)? -- What value will customers get from this integration, and why is it valuable to them? -- What specific data will your integration monitor, and what's the value of that data? +This check monitors [Milvus][1] through the Datadog Agent. It provides insights into your Milvus deployment's performance by collecting information about the latency and number of executions of individual operations. This integration also allows for monitoring the size and ressource allocation of your deployment. ## Setup @@ -20,7 +15,7 @@ No additional installation is needed on your server. ### Configuration -1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options. +1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your Milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options. 2. [Restart the Agent][5]. @@ -42,14 +37,12 @@ The Milvus integration does not include any events. The Milvus integration does not include any service checks. -See [service_checks.json][8] for a list of service checks provided by this integration. - ## Troubleshooting Need help? Contact [Datadog support][9]. -[1]: **LINK_TO_INTEGRATION_SITE** +[1]: https://milvus.io/ [2]: https://app.datadoghq.com/account/settings/agent/latest [3]: https://docs.datadoghq.com/agent/kubernetes/integrations/ [4]: https://github.com/DataDog/integrations-core/blob/master/milvus/datadog_checks/milvus/data/conf.yaml.example diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 5187ffe47bf77..2078dbbfe66e4 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -1,24 +1,24 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags -milvus.bf_search_cnt.bucket,count,,,,Number of bf search per request,0,milvus,,, -milvus.bf_search_cnt.count,count,,,,Number of bf search per request,0,milvus,,, -milvus.bf_search_cnt.sum,count,,,,Number of bf search per request,0,milvus,,, -milvus.bitset_ratio.bucket,count,,,,Bitset ratio,0,milvus,,, -milvus.bitset_ratio.count,count,,,,Bitset ratio,0,milvus,,, -milvus.bitset_ratio.sum,count,,,,Bitset ratio,0,milvus,,, +milvus.bf_search_cnt.bucket,count,,,,Histogram bucket for the number of brute-force searches per request,0,milvus,,, +milvus.bf_search_cnt.count,count,,,,Count aggregation of brute-force searches per request,0,milvus,,, +milvus.bf_search_cnt.sum,count,,,,Sum aggregation of brute-force searches per request,0,milvus,,, +milvus.bitset_ratio.bucket,count,,,,Bitset ratio histogram bucket,0,milvus,,, +milvus.bitset_ratio.count,count,,,,Count of bitset ratio calculations,0,milvus,,, +milvus.bitset_ratio.sum,count,,,,Sum of bitset ratios,0,milvus,,, milvus.build_info,gauge,,,,Build information of milvus,0,milvus,,, -milvus.build_latency.bucket,count,,,,Index build latency (s),0,milvus,,, -milvus.build_latency.count,count,,,,Index build latency (s),0,milvus,,, -milvus.build_latency.sum,count,,second,,Index build latency (s),0,milvus,,, -milvus.cache_hit_cnt.bucket,count,,,,Cache hit cnt per request,0,milvus,,, -milvus.cache_hit_cnt.count,count,,,,Cache hit cnt per request,0,milvus,,, -milvus.cache_hit_cnt.sum,count,,,,Cache hit cnt per request,0,milvus,,, +milvus.build_latency.bucket,count,,,,Index build latency histogram bucket,0,milvus,,, +milvus.build_latency.count,count,,,,Count of index builds,0,milvus,,, +milvus.build_latency.sum,count,,second,,Sum of index build latencies(s),0,milvus,,, +milvus.cache_hit_cnt.bucket,count,,,,Histogram bucket for the cache hit count per request,0,milvus,,, +milvus.cache_hit_cnt.count,count,,,,Count aggregation of cache hit count per request,0,milvus,,, +milvus.cache_hit_cnt.sum,count,,,,Sum aggregation of cache hit count per request,0,milvus,,, milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,,, -milvus.cgo.cgo_duration_seconds.bucket,count,,,,Histogram of cgo call duration in seconds.,0,milvus,,, -milvus.cgo.cgo_duration_seconds.count,count,,,,Histogram of cgo call duration in seconds.,0,milvus,,, -milvus.cgo.cgo_duration_seconds.sum,count,,second,,Histogram of cgo call duration in seconds.,0,milvus,,, -milvus.cgo.cgo_queue_duration_seconds.bucket,count,,,,Duration of cgo call in queue.,0,milvus,,, -milvus.cgo.cgo_queue_duration_seconds.count,count,,,,Duration of cgo call in queue.,0,milvus,,, -milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Duration of cgo call in queue.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.bucket,count,,,,Histogram bucket of cgo call duration in seconds,0,milvus,,, +milvus.cgo.cgo_duration_seconds.count,count,,,,Count of cgo calls,0,milvus,,, +milvus.cgo.cgo_duration_seconds.sum,count,,second,,Sum of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.bucket,count,,,,Histogram bucket for the duration of cgo calls in queue,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.count,count,,,,Counnt of cgo calls in queue,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Sum of the duration of cgo calls in queue,0,milvus,,, milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in unix seconds,0,milvus,,, milvus.datacoord.collection_num,gauge,,,,Number of collections,0,milvus,,, @@ -32,17 +32,17 @@ milvus.datacoord.segment_num,gauge,,,,Number of segments,0,milvus,,, milvus.datacoord.stored.binlog_size,gauge,,byte,,Binlog size of healthy segments,0,milvus,,, milvus.datacoord.stored.index_files_size,gauge,,byte,,Index files size of the segments,0,milvus,,, milvus.datacoord.stored.rows_num,gauge,,,,Number of stored rows of healthy segment,0,milvus,,, -milvus.datacoord.task_execute_max_latency.bucket,count,,,,Latency of task execute operation,0,milvus,,, -milvus.datacoord.task_execute_max_latency.count,count,,,,Latency of task execute operation,0,milvus,,, -milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,Latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.bucket,count,,,,Histogram bucket for the latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.count,count,,,,Count of task execute operations,0,milvus,,, +milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,Sum of the latency of task execute operations,0,milvus,,, milvus.datacoord.watched_dml_chanel_num,gauge,,,,The num of dml channel watched by datanode,0,milvus,,, milvus.datanode.autoflush_buffer_op.count,count,,,,Count of auto flush buffer operations,0,milvus,,, -milvus.datanode.consume.bytes.count,count,,,,,0,milvus,,, +milvus.datanode.consume.bytes.count,count,,,,Count of consumed bytes,0,milvus,,, milvus.datanode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, -milvus.datanode.encode_buffer_latency.bucket,count,,,,Latency of encode buffer data,0,milvus,,, -milvus.datanode.encode_buffer_latency.count,count,,,,Latency of encode buffer data,0,milvus,,, -milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,Latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.bucket,count,,,,Histogram bucket for the latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.count,count,,,,Count of encode buffer data operations,0,milvus,,, +milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,Sum of the latency of encode buffer data operations,0,milvus,,, milvus.datanode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, milvus.datanode.flush.buffer_op.count,count,,,,Count of flush buffer operations,0,milvus,,, milvus.datanode.flush.req.count,count,,,,Count of flush request,0,milvus,,, @@ -50,39 +50,39 @@ milvus.datanode.flushed_data.rows.count,count,,,,Num of rows flushed to storage, milvus.datanode.flushed_data.size.count,count,,byte,,Byte size of data flushed to storage,0,milvus,,, milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, milvus.datanode.msg.rows.count,count,,,,Count of rows consumed from msgStream,0,milvus,,, -milvus.datanode.save_latency.bucket,count,,,,Latency of saving flush data to storage,0,milvus,,, -milvus.datanode.save_latency.count,count,,,,Latency of saving flush data to storage,0,milvus,,, -milvus.datanode.save_latency.sum,count,,millisecond,,Latency of saving flush data to storage,0,milvus,,, -milvus.diskann.range_search_iters.bucket,count,,,,DISKANN range search iterations,0,milvus,,, -milvus.diskann.range_search_iters.count,count,,,,DISKANN range search iterations,0,milvus,,, -milvus.diskann.range_search_iters.sum,count,,,,DISKANN range search iterations,0,milvus,,, -milvus.diskann.search_hops.bucket,count,,,,DISKANN search hops,0,milvus,,, -milvus.diskann.search_hops.count,count,,,,DISKANN search hops,0,milvus,,, -milvus.diskann.search_hops.sum,count,,,,DISKANN search hops,0,milvus,,, -milvus.diskann_bitset_ratio.bucket,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, -milvus.diskann_bitset_ratio.count,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, -milvus.diskann_bitset_ratio.sum,count,,,,DISKANN bitset ratio for search and range search,0,milvus,,, -milvus.exec_latency.bucket,count,,,,Execute latency per request,0,milvus,,, -milvus.exec_latency.count,count,,,,Execute latency per request,0,milvus,,, -milvus.exec_latency.sum,count,,,,Execute latency per request,0,milvus,,, -milvus.filter.connectivity_ratio.bucket,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.connectivity_ratio.count,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.connectivity_ratio.sum,count,,,,Avg connectivity ratio set under filtering per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.bucket,count,,,,Avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.count,count,,,,Avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.activated_fields_cnt.sum,count,,,,Avg mv activated fields per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.bucket,count,,,,Mv change base cnt per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.count,count,,,,Mv change base cnt per request,0,milvus,,, -milvus.filter.mv.change_base_cnt.sum,count,,,,Mv change base cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.bucket,count,,,,Mv only cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.count,count,,,,Mv only cnt per request,0,milvus,,, -milvus.filter.mv.only_cnt.sum,count,,,,Mv only cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,Mv supplement ep from bitset boolean cnt per request,0,milvus,,, -milvus.flushed_segment_file_num.bucket,count,,,,The num of files for flushed segment,0,milvus,,, -milvus.flushed_segment_file_num.count,count,,,,The num of files for flushed segment,0,milvus,,, -milvus.flushed_segment_file_num.sum,count,,,,The num of files for flushed segment,0,milvus,,, +milvus.datanode.save_latency.bucket,count,,,,Histogram bucket for the latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.count,count,,,,Count of saving flush data to storage events,0,milvus,,, +milvus.datanode.save_latency.sum,count,,millisecond,,Sum of the latency of saving flush data to storage events,0,milvus,,, +milvus.diskann.range_search_iters.bucket,count,,,,Histogram bucket for DISKANN range search iterations,0,milvus,,, +milvus.diskann.range_search_iters.count,count,,,,Count of DISKANN range search events,0,milvus,,, +milvus.diskann.range_search_iters.sum,count,,,,Sum of DISKANN range search iterations,0,milvus,,, +milvus.diskann.search_hops.bucket,count,,,,Histogram for DISKANN search hops,0,milvus,,, +milvus.diskann.search_hops.count,count,,,,Count of DISKANN search hop events,0,milvus,,, +milvus.diskann.search_hops.sum,count,,,,Sum of DISKANN search hops,0,milvus,,, +milvus.diskann_bitset_ratio.bucket,count,,,,Histogram bucket for the DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.count,count,,,,Count of the DISKANN bitset ratio operations for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.sum,count,,,,Sum of the DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.exec_latency.bucket,count,,,,Histogram bucket for the execute latency per request,0,milvus,,, +milvus.exec_latency.count,count,,,,Count aggregation of the execute latency per request,0,milvus,,, +milvus.exec_latency.sum,count,,,,Sum aggregation of the execute latency per request,0,milvus,,, +milvus.filter.connectivity_ratio.bucket,count,,,,Histogram bucket for the average connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.count,count,,,,Count of the connectivity ratios set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.sum,count,,,,Sum of the average connectivity ratios set under filtering per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.bucket,count,,,,Histogram bucket of the average materialized view activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.count,count,,,,Count of the requests activating materialized view fields,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.sum,count,,,,Sum of the average materialized view activated fields per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.bucket,count,,,,Histogram bucket for the materialized view change base count per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.count,count,,,,Count of requests triggering materialized view change base events,0,milvus,,, +milvus.filter.mv.change_base_cnt.sum,count,,,,Sum of materialized view change base counts per request,0,milvus,,, +milvus.filter.mv.only_cnt.bucket,count,,,,Histogram bucket for the materialized view only count per request,0,milvus,,, +milvus.filter.mv.only_cnt.count,count,,,,Count of requests that trigger materialized view only events,0,milvus,,, +milvus.filter.mv.only_cnt.sum,count,,,,Sum of materialized view only counts,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,Histogram bucket for materialized view supplement entry point from bitset boolean count per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,Count of requests triggering a materialized view supplement entry point from bitset boolean event,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,Sum of materialized view supplement entry point from bitset boolean events,0,milvus,,, +milvus.flushed_segment_file_num.bucket,count,,,,Histogram bucket for the number of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.count,count,,,,Count of flushed segment events,0,milvus,,, +milvus.flushed_segment_file_num.sum,count,,,,Sum of the number of files for flushed segments,0,milvus,,, milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, @@ -112,69 +112,69 @@ milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stac milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,,, milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,,, milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,,, -milvus.graph_search_cnt.bucket,count,,,,Number of graph search per request,0,milvus,,, -milvus.graph_search_cnt.count,count,,,,Number of graph search per request,0,milvus,,, -milvus.graph_search_cnt.sum,count,,,,Number of graph search per request,0,milvus,,, -milvus.hnsw.bitset_ratio.bucket,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, -milvus.hnsw.bitset_ratio.count,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, -milvus.hnsw.bitset_ratio.sum,count,,,,HNSW bitset ratio for search and range search,0,milvus,,, -milvus.hnsw.search_hops.bucket,count,,,,HNSW search hops in layer 0,0,milvus,,, -milvus.hnsw.search_hops.count,count,,,,HNSW search hops in layer 0,0,milvus,,, -milvus.hnsw.search_hops.sum,count,,,,HNSW search hops in layer 0,0,milvus,,, -milvus.indexnode.build_index_latency.bucket,count,,,,Latency of build index for segment,0,milvus,,, -milvus.indexnode.build_index_latency.count,count,,,,Latency of build index for segment,0,milvus,,, -milvus.indexnode.build_index_latency.sum,count,,millisecond,,Latency of build index for segment,0,milvus,,, -milvus.indexnode.encode_index_latency.bucket,count,,,,Latency of encoding the index file,0,milvus,,, -milvus.indexnode.encode_index_latency.count,count,,,,Latency of encoding the index file,0,milvus,,, -milvus.indexnode.encode_index_latency.sum,count,,millisecond,,Latency of encoding the index file,0,milvus,,, +milvus.graph_search_cnt.bucket,count,,,,Histogram bucket fot the number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.count,count,,,,Count of requests that trigger a graph search,0,milvus,,, +milvus.graph_search_cnt.sum,count,,,,Sum of graph searches,0,milvus,,, +milvus.hnsw.bitset_ratio.bucket,count,,,,Histogram bucket of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.count,count,,,,Count aggregation of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.sum,count,,,,Sum aggregation of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.search_hops.bucket,count,,,,Histogram bucket of the HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.count,count,,,,Count aggregation of the HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.sum,count,,,,Sum aggregation of the HNSW search hops in layer 0,0,milvus,,, +milvus.indexnode.build_index_latency.bucket,count,,,,Histogram bucket of the latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.count,count,,,,Count of build index events,0,milvus,,, +milvus.indexnode.build_index_latency.sum,count,,millisecond,,Sum of the latency of build index events,0,milvus,,, +milvus.indexnode.encode_index_latency.bucket,count,,,,Histogram bucket of the latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.count,count,,,,Count of the index file encoding events,0,milvus,,, +milvus.indexnode.encode_index_latency.sum,count,,millisecond,,Sum of the latency of index file encoding events,0,milvus,,, milvus.indexnode.index.task.count,count,,,,Number of tasks that index node received,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.bucket,count,,,,Latency of index task in queue,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.count,count,,,,Latency of index task in queue,0,milvus,,, -milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,Latency of index task in queue,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.bucket,count,,,,Latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.count,count,,,,Latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,Latency of building the index by knowhere,0,milvus,,, -milvus.indexnode.save_index_latency.bucket,count,,,,Latency of saving the index file,0,milvus,,, -milvus.indexnode.save_index_latency.count,count,,,,Latency of saving the index file,0,milvus,,, -milvus.indexnode.save_index_latency.sum,count,,millisecond,,Latency of saving the index file,0,milvus,,, -milvus.internal.core_search_latency.bucket,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, -milvus.internal.core_search_latency.count,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, -milvus.internal.core_search_latency.sum,count,,,,[cpp]latency(us) of search on segment,0,milvus,,, -milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,[cpp]mmap allocated space stats,0,milvus,,, -milvus.internal.mmap.allocated_space_bytes.count,count,,,,[cpp]mmap allocated space stats,0,milvus,,, -milvus.internal.mmap.allocated_space_bytes.sum,count,,,,[cpp]mmap allocated space stats,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.bucket,count,,,,Histogram bucket for the latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.count,count,,,,Count of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,Sum of the latency of index task in queue,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.bucket,count,,,,Histogram bucket for the latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.count,count,,,,Count of index builds by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,Sum of the latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.save_index_latency.bucket,count,,,,Histogram bucket for the latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.count,count,,,,Count of index file save envents,0,milvus,,, +milvus.indexnode.save_index_latency.sum,count,,millisecond,,Sum of the latency of saving the index file,0,milvus,,, +milvus.internal.core_search_latency.bucket,count,,,,Histogram bucket for the [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.count,count,,,,Count aggregation of [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.sum,count,,microsecond,,Sum aggregation of [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,Histogram bucket for the [cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.count,count,,,,Count aggregation of [cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.sum,count,,,,Sum aggregation of [cpp]mmap allocated space stats,0,milvus,,, milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,,, -milvus.internal.storage.kv_size.bucket,count,,,,[cpp]kv size stats,0,milvus,,, -milvus.internal.storage.kv_size.count,count,,,,[cpp]kv size stats,0,milvus,,, -milvus.internal.storage.kv_size.sum,count,,,,[cpp]kv size stats,0,milvus,,, -milvus.internal.storage.load_duration.bucket,count,,,,[cpp]durations of load segment,0,milvus,,, -milvus.internal.storage.load_duration.count,count,,,,[cpp]durations of load segment,0,milvus,,, -milvus.internal.storage.load_duration.sum,count,,,,[cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.kv_size.bucket,count,,,,Histogram bucket for the [cpp]kv size stats,0,milvus,,, +milvus.internal.storage.kv_size.count,count,,,,Count aggregation of [cpp]kv size stats,0,milvus,,, +milvus.internal.storage.kv_size.sum,count,,,,Sum aggregation of [cpp]kv size stats,0,milvus,,, +milvus.internal.storage.load_duration.bucket,count,,,,Histogram bucket for the [cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.count,count,,,,Count aggregation of [cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.sum,count,,,,Sum aggregation of [cpp]durations of load segment,0,milvus,,, milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,,, -milvus.internal.storage.request_latency.bucket,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, -milvus.internal.storage.request_latency.count,count,,,,[cpp]request latency(ms) on the client side,0,milvus,,, -milvus.internal.storage.request_latency.sum,count,,millisecond,,[cpp]request latency(ms) on the client side,0,milvus,,, -milvus.io_cnt.bucket,count,,,,Io cnt per request,0,milvus,,, -milvus.io_cnt.count,count,,,,Io cnt per request,0,milvus,,, -milvus.io_cnt.sum,count,,,,Io cnt per request,0,milvus,,, -milvus.ivf_search_cnt.bucket,count,,,,Number of ivf search per request,0,milvus,,, -milvus.ivf_search_cnt.count,count,,,,Number of ivf search per request,0,milvus,,, -milvus.ivf_search_cnt.sum,count,,,,Number of ivf search per request,0,milvus,,, -milvus.load_latency.bucket,count,,,,Index load latency (ms),0,milvus,,, -milvus.load_latency.count,count,,,,Index load latency (ms),0,milvus,,, -milvus.load_latency.sum,count,,millisecond,,Index load latency (ms),0,milvus,,, -milvus.meta.kv_size.bucket,count,,,,Kv size stats,0,milvus,,, -milvus.meta.kv_size.count,count,,,,Kv size stats,0,milvus,,, -milvus.meta.kv_size.sum,count,,,,Kv size stats,0,milvus,,, +milvus.internal.storage.request_latency.bucket,count,,,,Histogram bucket for the [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.count,count,,,,Count aggregation of [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.sum,count,,millisecond,,Sum aggregation of [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.io_cnt.bucket,count,,,,Histogram bucket for the IO count per request,0,milvus,,, +milvus.io_cnt.count,count,,,,Count of requests triggering IO operations,0,milvus,,, +milvus.io_cnt.sum,count,,,,Sum of IO operation counts,0,milvus,,, +milvus.ivf_search_cnt.bucket,count,,,,Histogram bucket for the number of inverted file search per request,0,milvus,,, +milvus.ivf_search_cnt.count,count,,,,Count of requests trigerring an inverted file search,0,milvus,,, +milvus.ivf_search_cnt.sum,count,,,,Sum of inverted file searches,0,milvus,,, +milvus.load_latency.bucket,count,,,,Histogram bucket for the index load latency (ms),0,milvus,,, +milvus.load_latency.count,count,,,,Count of index load events,0,milvus,,, +milvus.load_latency.sum,count,,millisecond,,Sum of index load latency (ms),0,milvus,,, +milvus.meta.kv_size.bucket,count,,,,Histogram for the kv size stats,0,milvus,,, +milvus.meta.kv_size.count,count,,,,Count aggregation of the kv size stats,0,milvus,,, +milvus.meta.kv_size.sum,count,,,,Sum aggregation of the kv size stats,0,milvus,,, milvus.meta.op.count,count,,,,Count of meta operation,0,milvus,,, -milvus.meta.request_latency.bucket,count,,,,Request latency on the client side,0,milvus,,, -milvus.meta.request_latency.count,count,,,,Request latency on the client side,0,milvus,,, -milvus.meta.request_latency.sum,count,,millisecond,,Request latency on the client side,0,milvus,,, +milvus.meta.request_latency.bucket,count,,,,Histogram bucket for the request latency on the client side,0,milvus,,, +milvus.meta.request_latency.count,count,,,,Count aggregation of the request latency on the client side,0,milvus,,, +milvus.meta.request_latency.sum,count,,millisecond,,Sum aggregation of the request latency on the client side,0,milvus,,, milvus.msg_queue_consumer_num,gauge,,,,Number of consumers,0,milvus,,, milvus.msgstream.op.count,count,,,,Count of stream message operation,0,milvus,,, -milvus.msgstream.request_latency.bucket,count,,,,Request latency on the client side,0,milvus,,, -milvus.msgstream.request_latency.count,count,,,,Request latency on the client side,0,milvus,,, -milvus.msgstream.request_latency.sum,count,,millisecond,,Request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.bucket,count,,,,Histogram bucket for the request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.count,count,,,,Count aggregation of the request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.sum,count,,millisecond,,Sum aggregation of the request latency on the client side,0,milvus,,, milvus.num_node,gauge,,,,Number of nodes and coordinates,0,milvus,,, milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,,, milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,,, @@ -183,66 +183,66 @@ milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,m milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,,, milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,,, milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,,, -milvus.proxy.apply.pk_latency.bucket,count,,,,Latency that apply primary key,0,milvus,,, -milvus.proxy.apply.pk_latency.count,count,,,,Latency that apply primary key,0,milvus,,, -milvus.proxy.apply.pk_latency.sum,count,,millisecond,,Latency that apply primary key,0,milvus,,, -milvus.proxy.apply.timestamp_latency.bucket,count,,,,Latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.apply.timestamp_latency.count,count,,,,Latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,Latency that proxy apply timestamp,0,milvus,,, -milvus.proxy.assign_segmentID_latency.bucket,count,,,,Latency that proxy get segmentID from dataCoord,0,milvus,,, -milvus.proxy.assign_segmentID_latency.count,count,,,,Latency that proxy get segmentID from dataCoord,0,milvus,,, -milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,Latency that proxy get segmentID from dataCoord,0,milvus,,, -milvus.proxy.cache.hit.count,count,,,,count of cache hits/miss,0,milvus,,, -milvus.proxy.cache.update_latency.bucket,count,,,,Latency that proxy update cache when cache miss,0,milvus,,, -milvus.proxy.cache.update_latency.count,count,,,,Latency that proxy update cache when cache miss,0,milvus,,, -milvus.proxy.cache.update_latency.sum,count,,millisecond,,Latency that proxy update cache when cache miss,0,milvus,,, +milvus.proxy.apply.pk_latency.bucket,count,,,,Histogram bucket for the latency of "apply primary key" events,0,milvus,,, +milvus.proxy.apply.pk_latency.count,count,,,,Count of "apply primary key" events,0,milvus,,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,Sum of latencies for "apply primary key" events,0,milvus,,, +milvus.proxy.apply.timestamp_latency.bucket,count,,,,Histogram bucket for the latency of proxy "apply timestamp" events,0,milvus,,, +milvus.proxy.apply.timestamp_latency.count,count,,,,Count of proxy "apply timestamp" events,0,milvus,,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,Sum of latencies of proxy "apply timestamp" events,0,milvus,,, +milvus.proxy.assign_segmentID_latency.bucket,count,,,,Histogram bucket for the latency of proxy "get segmentID from dataCoord" events,0,milvus,,, +milvus.proxy.assign_segmentID_latency.count,count,,,,Count of proxy "get segmentID from dataCoord" events,0,milvus,,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,Sum of latencies for proxy "get segmentID from dataCoord" events,0,milvus,,, +milvus.proxy.cache.hit.count,count,,,,Count of cache hits/miss,0,milvus,,, +milvus.proxy.cache.update_latency.bucket,count,,,,Histogram bucket for the latency of proxy "update cache when cache miss" events,0,milvus,,, +milvus.proxy.cache.update_latency.count,count,,,,Count of proxy "update cache when cache miss" events,0,milvus,,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,Sum of latencies of proxy "update cache when cache miss",0,milvus,,, milvus.proxy.delete_vectors.count,count,,,,Counter of vectors successfully deleted,0,milvus,,, milvus.proxy.msgstream_obj_num,gauge,,,,Number of MsgStream objects per physical channel,0,milvus,,, -milvus.proxy.mutation_send_latency.bucket,count,,,,Latency that proxy send insert request to MsgStream,0,milvus,,, -milvus.proxy.mutation_send_latency.count,count,,,,Latency that proxy send insert request to MsgStream,0,milvus,,, -milvus.proxy.mutation_send_latency.sum,count,,millisecond,,Latency that proxy send insert request to MsgStream,0,milvus,,, +milvus.proxy.mutation_send_latency.bucket,count,,,,Histogram bucket for the latency of proxy "send insert request to MsgStream" events,0,milvus,,, +milvus.proxy.mutation_send_latency.count,count,,,,Count of proxy "send insert request to MsgStream" events,0,milvus,,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,Sum of latencies of proxy "send insert request to MsgStream" events,0,milvus,,, milvus.proxy.rate_limit_req.count,count,,,,Count of operation executed,0,milvus,,, milvus.proxy.report_value.count,count,,,,Report value about the request,0,milvus,,, milvus.proxy.req.count,count,,,,Count of operation executed,0,milvus,,, -milvus.proxy.req.in_queue_latency.bucket,count,,,,Latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.in_queue_latency.count,count,,,,Latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,Latency which request waits in the queue,0,milvus,,, -milvus.proxy.req.latency.bucket,count,,,,Latency of each request,0,milvus,,, -milvus.proxy.req.latency.count,count,,,,Latency of each request,0,milvus,,, -milvus.proxy.req.latency.sum,count,,millisecond,,Latency of each request,0,milvus,,, +milvus.proxy.req.in_queue_latency.bucket,count,,,,Histogram bucket for the time in queue of request,0,milvus,,, +milvus.proxy.req.in_queue_latency.count,count,,,,Count of request in queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,Sum of the time requests spend in queue,0,milvus,,, +milvus.proxy.req.latency.bucket,count,,,,Histogram bucket for the latency of each request,0,milvus,,, +milvus.proxy.req.latency.count,count,,,,Count of request latency events,0,milvus,,, +milvus.proxy.req.latency.sum,count,,millisecond,,Sum of latencies of requests,0,milvus,,, milvus.proxy.send_bytes.count,count,,,,Count of bytes sent back to sdk,0,milvus,,, -milvus.proxy.sq.decode_result_latency.bucket,count,,,,Latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.decode_result_latency.count,count,,,,Latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,Latency that proxy decodes the search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.bucket,count,,,,Latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.count,count,,,,Latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,Latency that proxy reduces search result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.bucket,count,,,,Latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.count,count,,,,Latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,Latency that proxy waits for the result,0,milvus,,, -milvus.proxy.sync_segment_request_length.bucket,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.sync_segment_request_length.count,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.sync_segment_request_length.sum,count,,,,The length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sq.decode_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "decode the search result" events,0,milvus,,, +milvus.proxy.sq.decode_result_latency.count,count,,,,Count of proxy "decode the search result" events,0,milvus,,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "decode the search result" events,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "reduces search result" events,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.count,count,,,,Count of proxy "reduces search result" events,0,milvus,,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "reduces search result" events,0,milvus,,, +milvus.proxy.sq.wait_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "waits for the result" events,0,milvus,,, +milvus.proxy.sq.wait_result_latency.count,count,,,,Count of proxy "waits for the result" events,0,milvus,,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "waits for the result" events,0,milvus,,, +milvus.proxy.sync_segment_request_length.bucket,count,,,,Histogram bucket for the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.count,count,,,,Count of assigning segments for insert events,0,milvus,,, +milvus.proxy.sync_segment_request_length.sum,count,,,,Sum of the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, milvus.proxy.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, -milvus.quant.compute_cnt.bucket,count,,,,Quant compute cnt per request,0,milvus,,, -milvus.quant.compute_cnt.count,count,,,,Quant compute cnt per request,0,milvus,,, -milvus.quant.compute_cnt.sum,count,,,,Quant compute cnt per request,0,milvus,,, +milvus.quant.compute_cnt.bucket,count,,,,Histogram bucket for the quant compute count per request,0,milvus,,, +milvus.quant.compute_cnt.count,count,,,,Count of requests triggering quant compute events,0,milvus,,, +milvus.quant.compute_cnt.sum,count,,,,Sum of quant compute event counts,0,milvus,,, milvus.querycoord.collection_num,gauge,,,,Number of collections,0,milvus,,, milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,Current target checkpoint timestamp in unix seconds,0,milvus,,, -milvus.querycoord.load.latency.bucket,count,,,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.bucket,count,,,,Histogram bucket Latency of load the entire collection,0,milvus,,, milvus.querycoord.load.latency.count,count,,,,Latency of load the entire collection,0,milvus,,, milvus.querycoord.load.latency.sum,count,,millisecond,,Latency of load the entire collection,0,milvus,,, milvus.querycoord.load.req.count,count,,,,Count of load request,0,milvus,,, milvus.querycoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, milvus.querycoord.querynode_num,gauge,,,,Number of QueryNodes managered by QueryCoord,0,milvus,,, -milvus.querycoord.release.latency.bucket,count,,,,Latency of release request,0,milvus,,, -milvus.querycoord.release.latency.count,count,,,,Latency of release request,0,milvus,,, -milvus.querycoord.release.latency.sum,count,,millisecond,,Latency of release request,0,milvus,,, +milvus.querycoord.release.latency.bucket,count,,,,Histogram bucket for the latency of release request,0,milvus,,, +milvus.querycoord.release.latency.count,count,,,,Count of release request events,0,milvus,,, +milvus.querycoord.release.latency.sum,count,,millisecond,,Sum of the latencies of release request events,0,milvus,,, milvus.querycoord.release.req.count,count,,,,Count of release request,0,milvus,,, milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,,, -milvus.querynode.apply_bf_latency.bucket,count,,,,Apply bf cost in ms,0,milvus,,, -milvus.querynode.apply_bf_latency.count,count,,,,Apply bf cost in ms,0,milvus,,, -milvus.querynode.apply_bf_latency.sum,count,,millisecond,,Apply bf cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.bucket,count,,,,Histogram bucket for the apply brute-force cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.count,count,,,,Count of apply brute-force events,0,milvus,,, +milvus.querynode.apply_bf_latency.sum,count,,millisecond,,Sum of the apply brute-force cost in ms,0,milvus,,, milvus.querynode.collection_num,gauge,,,,Number of collections loaded,0,milvus,,, milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,,, milvus.querynode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, @@ -250,115 +250,115 @@ milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per phy milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,Number of bytes evicted from disk cache,0,milvus,,, milvus.querynode.disk.cache.evict.count,count,,,,Number of segments evicted from disk cache,0,milvus,,, milvus.querynode.disk.cache.evict.duration.count,count,,,,Total time cost of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.bucket,count,,,,Global duration of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.count,count,,,,Global duration of evicting segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,Global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.bucket,count,,,,Histogram bucket for the global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.count,count,,,,Count of evicting segments from disk cache events,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,Sum of the global duration of evicting segments from disk cache,0,milvus,,, milvus.querynode.disk.cache.load.bytes.count,count,,byte,,Number of bytes loaded from disk cache,0,milvus,,, milvus.querynode.disk.cache.load.count,count,,,,Number of segments loaded from disk cache,0,milvus,,, milvus.querynode.disk.cache.load.duration.count,count,,,,Total time cost of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.bucket,count,,,,Global duration of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.count,count,,,,Global duration of loading segments from disk cache,0,milvus,,, -milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,Global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.bucket,count,,,,Histogram bucket for the global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.count,count,,,,Count of loading segments from disk cache events,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,Sum of the global duration of loading segments from disk cache,0,milvus,,, milvus.querynode.disk.used_size,gauge,,mebibyte,,Disk used size(MB),0,milvus,,, milvus.querynode.dml_vchannel_num,gauge,,,,Number of dmlChannels watched,0,milvus,,, milvus.querynode.entity.num,gauge,,,,"Number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, milvus.querynode.entity.size,gauge,,byte,,"Entities' memory size, clustered by collection and state",0,milvus,,, milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,,, milvus.querynode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, -milvus.querynode.forward_delete_latency.bucket,count,,,,Forward delete cost in ms,0,milvus,,, -milvus.querynode.forward_delete_latency.count,count,,,,Forward delete cost in ms,0,milvus,,, -milvus.querynode.forward_delete_latency.sum,count,,millisecond,,Forward delete cost in ms,0,milvus,,, -milvus.querynode.load.index_latency.bucket,count,,,,"Latency of load per segment's index, in milliseconds",0,milvus,,, -milvus.querynode.load.index_latency.count,count,,,,"Latency of load per segment's index, in milliseconds",0,milvus,,, -milvus.querynode.load.index_latency.sum,count,,millisecond,,"Latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.forward_delete_latency.bucket,count,,,,Histogram bucket for the forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.count,count,,,,Count of forward delete events,0,milvus,,, +milvus.querynode.forward_delete_latency.sum,count,,millisecond,,Sum of the forward delete cost in ms,0,milvus,,, +milvus.querynode.load.index_latency.bucket,count,,,,"Histogram bucket for the latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.count,count,,,,"Count of load per segment's index events",0,milvus,,, +milvus.querynode.load.index_latency.sum,count,,millisecond,,"Sum of the latencies of load per segment's index events, in milliseconds",0,milvus,,, milvus.querynode.load.segment.concurrency,gauge,,,,Number of concurrent loading segments in QueryNode,0,milvus,,, -milvus.querynode.load.segment.latency.bucket,count,,,,Latency of load per segment,0,milvus,,, -milvus.querynode.load.segment.latency.count,count,,,,Latency of load per segment,0,milvus,,, -milvus.querynode.load.segment.latency.sum,count,,millisecond,,Latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.bucket,count,,,,Histogram bucket for the latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.count,count,,,,Count of load per segment events,0,milvus,,, +milvus.querynode.load.segment.latency.sum,count,,millisecond,,Sum of the latencies of load per segment events,0,milvus,,, milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, milvus.querynode.partition_num,gauge,,,,Number of partitions loaded,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.bucket,count,,,,Process insert or delete cost in ms,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.count,count,,,,Process insert or delete cost in ms,0,milvus,,, -milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,Process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.bucket,count,,,,Histogram bucket for the process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.count,count,,,,Count of process insert or delete events,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,Sum of process insert or delete cost in ms,0,milvus,,, milvus.querynode.read_task.concurrency,gauge,,,,Number of concurrent executing read tasks in QueryNode,0,milvus,,, milvus.querynode.read_task.ready_len,gauge,,,,Number of ready read tasks in readyQueue,0,milvus,,, milvus.querynode.read_task.unsolved_len,gauge,,,,Number of unsolved read tasks in unsolvedQueue,0,milvus,,, -milvus.querynode.search.group.nq.bucket,count,,,,The number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.nq.count,count,,,,The number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.nq.sum,count,,,,The number of queries of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.bucket,count,,,,The number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.count,count,,,,The number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.size.sum,count,,,,The number of tasks of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.bucket,count,,,,The topK of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.count,count,,,,The topK of each grouped search task,0,milvus,,, -milvus.querynode.search.group.topk.sum,count,,,,The topK of each grouped search task,0,milvus,,, -milvus.querynode.search.nq.bucket,count,,,,The number of queries of each search task,0,milvus,,, -milvus.querynode.search.nq.count,count,,,,The number of queries of each search task,0,milvus,,, -milvus.querynode.search.nq.sum,count,,,,The number of queries of each search task,0,milvus,,, -milvus.querynode.search.topk.bucket,count,,,,The top of each search task,0,milvus,,, -milvus.querynode.search.topk.count,count,,,,The top of each search task,0,milvus,,, -milvus.querynode.search.topk.sum,count,,,,The top of each search task,0,milvus,,, +milvus.querynode.search.group.nq.bucket,count,,,,Histogram bucket for the number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.nq.sum,count,,,,Sum of the number of queries of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.size.bucket,count,,,,Histogram bucket for the number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.size.sum,count,,,,Sum of the number of tasks of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.topk.bucket,count,,,,Histogram bucket for the topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.topk.sum,count,,,,Sum of the topK of grouped search tasks,0,milvus,,, +milvus.querynode.search.nq.bucket,count,,,,Histogram bucket for the number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.count,count,,,,Count of search tasks,0,milvus,,, +milvus.querynode.search.nq.sum,count,,,,Sum of the number of queries of search tasks,0,milvus,,, +milvus.querynode.search.topk.bucket,count,,,,Histogram bucket for the top of each search task,0,milvus,,, +milvus.querynode.search.topk.count,count,,,,Count of search tasks,0,milvus,,, +milvus.querynode.search.topk.sum,count,,,,Sum of the top of search tasks,0,milvus,,, milvus.querynode.segment.access.count,count,,,,Number of segments accessed,0,milvus,,, milvus.querynode.segment.access.duration.count,count,,,,Total time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.bucket,count,,,,Global time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.count,count,,,,Global time cost of accessing segments,0,milvus,,, -milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,Global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.bucket,count,,,,Histogram bucket for the global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.count,count,,,,Count of accessing segments events,0,milvus,,, +milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,Sum of the global time cost of accessing segments,0,milvus,,, milvus.querynode.segment.access.wait_cache.count,count,,,,Number of segments waiting for loading access,0,milvus,,, milvus.querynode.segment.access.wait_cache.duration.count,count,,,,Total time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,,,Global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.count,count,,,,Global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,Global time cost of waiting for loading access,0,milvus,,, -milvus.querynode.segment.latency_per_vector.bucket,count,,,,One vector's search latency per segment,0,milvus,,, -milvus.querynode.segment.latency_per_vector.count,count,,,,One vector's search latency per segment,0,milvus,,, -milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,One vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,,,Histogram bucket for the global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.count,count,,,,Count of waiting for loading access events,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,Sum of the global time cost of waiting for loading access events,0,milvus,,, +milvus.querynode.segment.latency_per_vector.bucket,count,,,,Histogram bucket for one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.count,count,,,,Count aggregation of one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,Sum aggregation of one vector's search latency per segment,0,milvus,,, milvus.querynode.segment.num,gauge,,,,"Number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,,, -milvus.querynode.sq.core_latency.bucket,count,,,,Latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.core_latency.count,count,,,,Latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.core_latency.sum,count,,millisecond,,Latency of search or query latency in segcore,0,milvus,,, -milvus.querynode.sq.queue.latency.bucket,count,,,,Latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.latency.count,count,,,,Latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.latency.sum,count,,millisecond,,Latency of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.bucket,count,,,,Latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.count,count,,,,Latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,Latency per user of search or query in queue,0,milvus,,, -milvus.querynode.sq.reduce_latency.bucket,count,,,,Latency of reduce search or query result,0,milvus,,, -milvus.querynode.sq.reduce_latency.count,count,,,,Latency of reduce search or query result,0,milvus,,, -milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,Latency of reduce search or query result,0,milvus,,, +milvus.querynode.sq.core_latency.bucket,count,,,,Histogram bucket for the latency of searches or queries in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.count,count,,,,Count of search or query events in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.sum,count,,millisecond,,Sum of latencies of searches or queries in segcore,0,milvus,,, +milvus.querynode.sq.queue.latency.bucket,count,,,,Histogram bucket for the latency of searches or queries in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.count,count,,,,Count of search or query events,0,milvus,,, +milvus.querynode.sq.queue.latency.sum,count,,millisecond,,Sum of latencies of searches or queries in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.bucket,count,,,,Histgram bucket for the latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.count,count,,,,Count of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,Sum of latencies per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.reduce_latency.bucket,count,,,,Histogram bucket for the latencies of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.count,count,,,,Count of reduce search or query result events,0,milvus,,, +milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,Sum of latencies of reduce search or query result,0,milvus,,, milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,,, -milvus.querynode.sq.req.latency.bucket,count,,,,Latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.req.latency.count,count,,,,Latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.req.latency.sum,count,,millisecond,,Latency of Search or query requests,0,milvus,,, -milvus.querynode.sq.segment_latency.bucket,count,,,,Latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.segment_latency.count,count,,,,Latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.segment_latency.sum,count,,millisecond,,Latency of search or query per segment,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.bucket,count,,,,Latency of search or query to wait for tsafe,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.count,count,,,,Latency of search or query to wait for tsafe,0,milvus,,, -milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,Latency of search or query to wait for tsafe,0,milvus,,, +milvus.querynode.sq.req.latency.bucket,count,,,,Histogram bucket for the latency of search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.count,count,,,,Count of search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.sum,count,,millisecond,,Sum of latencies of search or query requests,0,milvus,,, +milvus.querynode.sq.segment_latency.bucket,count,,,,Histogram bucket for the latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.count,count,,,,Count of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.sum,count,,millisecond,,Sum of latencies of search or query per segment,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.bucket,count,,,,Histogram bucket for the time that search or query waits for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.count,count,,,,Count of occurrences of search or query waiting for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,Sum of the time that search or query wait for tsafe,0,milvus,,, milvus.querynode.wait_processing_msg,gauge,,,,Count of wait processing msg,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Latency of watch dml channel,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.count,count,,,,Latency of watch dml channel,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Latency of watch dml channel,0,milvus,,, -milvus.queue.latency.bucket,count,,,,Queue latency per request,0,milvus,,, -milvus.queue.latency.count,count,,,,Queue latency per request,0,milvus,,, -milvus.queue.latency.sum,count,,,,Queue latency per request,0,milvus,,, -milvus.range_search_latency.bucket,count,,,,Range search latency (ms),0,milvus,,, -milvus.range_search_latency.count,count,,,,Range search latency (ms),0,milvus,,, -milvus.range_search_latency.sum,count,,millisecond,,Range search latency (ms),0,milvus,,, -milvus.raw_compute_cnt.bucket,count,,,,Raw compute cnt per request,0,milvus,,, -milvus.raw_compute_cnt.count,count,,,,Raw compute cnt per request,0,milvus,,, -milvus.raw_compute_cnt.sum,count,,,,Raw compute cnt per request,0,milvus,,, -milvus.re_search_cnt.bucket,count,,,,Number of fallback search per request,0,milvus,,, -milvus.re_search_cnt.count,count,,,,Number of fallback search per request,0,milvus,,, -milvus.re_search_cnt.sum,count,,,,Number of fallback search per request,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Histogram bucket for the latencies of watch dml channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.count,count,,,,Count of watch dml channel events,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Sum of the latencies of watch dml channel events,0,milvus,,, +milvus.queue.latency.bucket,count,,,,Histogram bucket for the queue latency per request,0,milvus,,, +milvus.queue.latency.count,count,,,,Count of request with queue latency,0,milvus,,, +milvus.queue.latency.sum,count,,,,Sum of queue latencies,0,milvus,,, +milvus.range_search_latency.bucket,count,,,,Histogram bucket for range search latency (ms),0,milvus,,, +milvus.range_search_latency.count,count,,,,Count of range search operations,0,milvus,,, +milvus.range_search_latency.sum,count,,millisecond,,Sum of range search latencies (ms),0,milvus,,, +milvus.raw_compute_cnt.bucket,count,,,,Histogram bucket for the raw compute count per request,0,milvus,,, +milvus.raw_compute_cnt.count,count,,,,Count of request that trigger raw compute operations,0,milvus,,, +milvus.raw_compute_cnt.sum,count,,,,Sum of raw compute operations,0,milvus,,, +milvus.re_search_cnt.bucket,count,,,,Histogram bucket for the number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.count,count,,,,Count of requests triggering fallback search operations,0,milvus,,, +milvus.re_search_cnt.sum,count,,,,Sum of fallback searches,0,milvus,,, milvus.rootcoord.collection_num,gauge,,,,Number of collections,0,milvus,,, milvus.rootcoord.credential_num,gauge,,,,Number of credentials,0,milvus,,, milvus.rootcoord.ddl_req.count,count,,,,Count of DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.bucket,count,,,,Latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.count,count,,,,Latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,Latency of each DDL operations,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,,,Latency of each DDL operations in queue,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.count,count,,,,Latency of each DDL operations in queue,0,milvus,,, -milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,Latency of each DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency.bucket,count,,,,Histogram bucket for the latency of each DDL operation,0,milvus,,, +milvus.rootcoord.ddl_req.latency.count,count,,,,Count of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,Sum of the latencies of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,,,Histogram bucket for the latencies of each DDL operation in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.count,count,,,,Count of DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,Sum of the latencies of DDL operations in queue,0,milvus,,, milvus.rootcoord.dml_channel_num,gauge,,,,Number of DML channels,0,milvus,,, milvus.rootcoord.entity_num,gauge,,,,"Number of entities, clustered by collection and their status(loaded/total)",0,milvus,,, milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,,, @@ -370,18 +370,18 @@ milvus.rootcoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, milvus.rootcoord.proxy_num,gauge,,,,Number of proxy nodes managered by rootcoord,0,milvus,,, milvus.rootcoord.qn_mem_high_water_level,gauge,,,,Querynode memory high water level,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.bucket,count,,,,Latency of synchronizing timetick message,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.count,count,,,,Latency of synchronizing timetick message,0,milvus,,, -milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,Latency of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.bucket,count,,,,Histogram bucket for the latencies of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.count,count,,,,Count of synchronizing timetick message events,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,Sum of latencies of synchronizing timetick message events,0,milvus,,, milvus.rootcoord.timestamp,gauge,,,,Latest timestamp allocated in memory,0,milvus,,, milvus.rootcoord.timestamp_saved,gauge,,,,Timestamp saved in meta storage,0,milvus,,, -milvus.runtime_info,gauge,,,,Runtime information of milvus,0,milvus,,, -milvus.search.latency.bucket,count,,,,Search latency (ms),0,milvus,,, -milvus.search.latency.count,count,,,,Search latency (ms),0,milvus,,, -milvus.search.latency.sum,count,,millisecond,,Search latency (ms),0,milvus,,, -milvus.search.topk.bucket,count,,,,Search topk,0,milvus,,, -milvus.search.topk.count,count,,,,Search topk,0,milvus,,, -milvus.search.topk.sum,count,,,,Search topk,0,milvus,,, +milvus.runtime_info,gauge,,,,Runtime information of Milvus,0,milvus,,, +milvus.search.latency.bucket,count,,,,Histogram bucket for search latencies (ms),0,milvus,,, +milvus.search.latency.count,count,,,,Count of search events,0,milvus,,, +milvus.search.latency.sum,count,,millisecond,,Sum of search latencies (ms),0,milvus,,, +milvus.search.topk.bucket,count,,,,Histogram bucket for search topk,0,milvus,,, +milvus.search.topk.count,count,,,,Count aggregation of search topk,0,milvus,,, +milvus.search.topk.sum,count,,,,Sum aggregation of search topk,0,milvus,,, milvus.storage.kv_size,gauge,,,,Kv size stats,0,milvus,,, milvus.storage.op.count,count,,,,Count of persistent data operation,0,milvus,,, milvus.storage.request_latency,gauge,,,,Request latency on the client side,0,milvus,,, From cf5d4e9fab086e52e0aa7282c41b76a6dcb91362 Mon Sep 17 00:00:00 2001 From: dkirov-dd <166512750+dkirov-dd@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:13:34 +0100 Subject: [PATCH 27/31] Capitalize unix in metadata description Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com> --- milvus/metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 2078dbbfe66e4..58757fbc376f3 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -20,7 +20,7 @@ milvus.cgo.cgo_queue_duration_seconds.bucket,count,,,,Histogram bucket for the d milvus.cgo.cgo_queue_duration_seconds.count,count,,,,Counnt of cgo calls in queue,0,milvus,,, milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Sum of the duration of cgo calls in queue,0,milvus,,, milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, -milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in unix seconds,0,milvus,,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in Unix seconds,0,milvus,,, milvus.datacoord.collection_num,gauge,,,,Number of collections,0,milvus,,, milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, milvus.datacoord.datanode_num,gauge,,,,Number of data nodes,0,milvus,,, From ef3ee9276b2d5584d862709e0541904ee7f09032 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 23 Dec 2024 13:32:34 +0200 Subject: [PATCH 28/31] Incorporate documentation suggestions --- milvus/metadata.csv | 92 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 58757fbc376f3..9725cfdb508e5 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -22,7 +22,7 @@ milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Sum of the duration of milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in Unix seconds,0,milvus,,, milvus.datacoord.collection_num,gauge,,,,Number of collections,0,milvus,,, -milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, milvus.datacoord.datanode_num,gauge,,,,Number of data nodes,0,milvus,,, milvus.datacoord.import_tasks,gauge,,,,The import tasks grouping by type and state,0,milvus,,, milvus.datacoord.index.node_num,gauge,,,,Number of IndexNodes managed by IndexCoord,0,milvus,,, @@ -35,11 +35,11 @@ milvus.datacoord.stored.rows_num,gauge,,,,Number of stored rows of healthy segme milvus.datacoord.task_execute_max_latency.bucket,count,,,,Histogram bucket for the latency of task execute operation,0,milvus,,, milvus.datacoord.task_execute_max_latency.count,count,,,,Count of task execute operations,0,milvus,,, milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,Sum of the latency of task execute operations,0,milvus,,, -milvus.datacoord.watched_dml_chanel_num,gauge,,,,The num of dml channel watched by datanode,0,milvus,,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,The num of data manipulation language channel watched by datanode,0,milvus,,, milvus.datanode.autoflush_buffer_op.count,count,,,,Count of auto flush buffer operations,0,milvus,,, milvus.datanode.consume.bytes.count,count,,,,Count of consumed bytes,0,milvus,,, milvus.datanode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, -milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, milvus.datanode.encode_buffer_latency.bucket,count,,,,Histogram bucket for the latency of encode buffer data,0,milvus,,, milvus.datanode.encode_buffer_latency.count,count,,,,Count of encode buffer data operations,0,milvus,,, milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,Sum of the latency of encode buffer data operations,0,milvus,,, @@ -112,7 +112,7 @@ milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stac milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,,, milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,,, milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,,, -milvus.graph_search_cnt.bucket,count,,,,Histogram bucket fot the number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.bucket,count,,,,Histogram bucket for the number of graph search per request,0,milvus,,, milvus.graph_search_cnt.count,count,,,,Count of requests that trigger a graph search,0,milvus,,, milvus.graph_search_cnt.sum,count,,,,Sum of graph searches,0,milvus,,, milvus.hnsw.bitset_ratio.bucket,count,,,,Histogram bucket of the HNSW bitset ratio for search and range search,0,milvus,,, @@ -144,9 +144,9 @@ milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,Histogram bucket for milvus.internal.mmap.allocated_space_bytes.count,count,,,,Count aggregation of [cpp]mmap allocated space stats,0,milvus,,, milvus.internal.mmap.allocated_space_bytes.sum,count,,,,Sum aggregation of [cpp]mmap allocated space stats,0,milvus,,, milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,,, -milvus.internal.storage.kv_size.bucket,count,,,,Histogram bucket for the [cpp]kv size stats,0,milvus,,, -milvus.internal.storage.kv_size.count,count,,,,Count aggregation of [cpp]kv size stats,0,milvus,,, -milvus.internal.storage.kv_size.sum,count,,,,Sum aggregation of [cpp]kv size stats,0,milvus,,, +milvus.internal.storage.kv_size.bucket,count,,,,Histogram bucket for the [cpp]key-value size stats,0,milvus,,, +milvus.internal.storage.kv_size.count,count,,,,Count aggregation of [cpp]key-value size stats,0,milvus,,, +milvus.internal.storage.kv_size.sum,count,,,,Sum aggregation of [cpp]key-value size stats,0,milvus,,, milvus.internal.storage.load_duration.bucket,count,,,,Histogram bucket for the [cpp]durations of load segment,0,milvus,,, milvus.internal.storage.load_duration.count,count,,,,Count aggregation of [cpp]durations of load segment,0,milvus,,, milvus.internal.storage.load_duration.sum,count,,,,Sum aggregation of [cpp]durations of load segment,0,milvus,,, @@ -163,9 +163,9 @@ milvus.ivf_search_cnt.sum,count,,,,Sum of inverted file searches,0,milvus,,, milvus.load_latency.bucket,count,,,,Histogram bucket for the index load latency (ms),0,milvus,,, milvus.load_latency.count,count,,,,Count of index load events,0,milvus,,, milvus.load_latency.sum,count,,millisecond,,Sum of index load latency (ms),0,milvus,,, -milvus.meta.kv_size.bucket,count,,,,Histogram for the kv size stats,0,milvus,,, -milvus.meta.kv_size.count,count,,,,Count aggregation of the kv size stats,0,milvus,,, -milvus.meta.kv_size.sum,count,,,,Sum aggregation of the kv size stats,0,milvus,,, +milvus.meta.kv_size.bucket,count,,,,Histogram for the key-value size stats,0,milvus,,, +milvus.meta.kv_size.count,count,,,,Count aggregation of the key-value size stats,0,milvus,,, +milvus.meta.kv_size.sum,count,,,,Sum aggregation of the key-value size stats,0,milvus,,, milvus.meta.op.count,count,,,,Count of meta operation,0,milvus,,, milvus.meta.request_latency.bucket,count,,,,Histogram bucket for the request latency on the client side,0,milvus,,, milvus.meta.request_latency.count,count,,,,Count aggregation of the request latency on the client side,0,milvus,,, @@ -180,27 +180,27 @@ milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent i milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,,, milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,,, milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,,, -milvus.process.start_time_seconds,gauge,,,,Start time of the process since unix epoch in seconds.,0,milvus,,, +milvus.process.start_time_seconds,gauge,,,,Start time of the process since Unix epoch in seconds.,0,milvus,,, milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,,, milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,,, -milvus.proxy.apply.pk_latency.bucket,count,,,,Histogram bucket for the latency of "apply primary key" events,0,milvus,,, -milvus.proxy.apply.pk_latency.count,count,,,,Count of "apply primary key" events,0,milvus,,, -milvus.proxy.apply.pk_latency.sum,count,,millisecond,,Sum of latencies for "apply primary key" events,0,milvus,,, -milvus.proxy.apply.timestamp_latency.bucket,count,,,,Histogram bucket for the latency of proxy "apply timestamp" events,0,milvus,,, -milvus.proxy.apply.timestamp_latency.count,count,,,,Count of proxy "apply timestamp" events,0,milvus,,, -milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,Sum of latencies of proxy "apply timestamp" events,0,milvus,,, -milvus.proxy.assign_segmentID_latency.bucket,count,,,,Histogram bucket for the latency of proxy "get segmentID from dataCoord" events,0,milvus,,, -milvus.proxy.assign_segmentID_latency.count,count,,,,Count of proxy "get segmentID from dataCoord" events,0,milvus,,, -milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,Sum of latencies for proxy "get segmentID from dataCoord" events,0,milvus,,, +milvus.proxy.apply.pk_latency.bucket,count,,,,"Histogram bucket for the latency of 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.pk_latency.count,count,,,,"Count of 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,"Sum of latencies for 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.count,count,,,,"Count of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'get segmentID from dataCoord' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.count,count,,,,"Count of proxy 'get segmentID from dataCoord' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'get segmentID from dataCoord' events",0,milvus,,, milvus.proxy.cache.hit.count,count,,,,Count of cache hits/miss,0,milvus,,, -milvus.proxy.cache.update_latency.bucket,count,,,,Histogram bucket for the latency of proxy "update cache when cache miss" events,0,milvus,,, -milvus.proxy.cache.update_latency.count,count,,,,Count of proxy "update cache when cache miss" events,0,milvus,,, -milvus.proxy.cache.update_latency.sum,count,,millisecond,,Sum of latencies of proxy "update cache when cache miss",0,milvus,,, +milvus.proxy.cache.update_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'update cache when cache miss' events",0,milvus,,, +milvus.proxy.cache.update_latency.count,count,,,,"Count of proxy 'update cache when cache miss' events",0,milvus,,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'update cache when cache miss'",0,milvus,,, milvus.proxy.delete_vectors.count,count,,,,Counter of vectors successfully deleted,0,milvus,,, milvus.proxy.msgstream_obj_num,gauge,,,,Number of MsgStream objects per physical channel,0,milvus,,, -milvus.proxy.mutation_send_latency.bucket,count,,,,Histogram bucket for the latency of proxy "send insert request to MsgStream" events,0,milvus,,, -milvus.proxy.mutation_send_latency.count,count,,,,Count of proxy "send insert request to MsgStream" events,0,milvus,,, -milvus.proxy.mutation_send_latency.sum,count,,millisecond,,Sum of latencies of proxy "send insert request to MsgStream" events,0,milvus,,, +milvus.proxy.mutation_send_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'send insert request to MsgStream' events",0,milvus,,, +milvus.proxy.mutation_send_latency.count,count,,,,"Count of proxy 'send insert request to MsgStream' events",0,milvus,,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'send insert request to MsgStream' events",0,milvus,,, milvus.proxy.rate_limit_req.count,count,,,,Count of operation executed,0,milvus,,, milvus.proxy.report_value.count,count,,,,Report value about the request,0,milvus,,, milvus.proxy.req.count,count,,,,Count of operation executed,0,milvus,,, @@ -211,24 +211,24 @@ milvus.proxy.req.latency.bucket,count,,,,Histogram bucket for the latency of eac milvus.proxy.req.latency.count,count,,,,Count of request latency events,0,milvus,,, milvus.proxy.req.latency.sum,count,,millisecond,,Sum of latencies of requests,0,milvus,,, milvus.proxy.send_bytes.count,count,,,,Count of bytes sent back to sdk,0,milvus,,, -milvus.proxy.sq.decode_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "decode the search result" events,0,milvus,,, -milvus.proxy.sq.decode_result_latency.count,count,,,,Count of proxy "decode the search result" events,0,milvus,,, -milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "decode the search result" events,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "reduces search result" events,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.count,count,,,,Count of proxy "reduces search result" events,0,milvus,,, -milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "reduces search result" events,0,milvus,,, -milvus.proxy.sq.wait_result_latency.bucket,count,,,,Histogram bucket for the latency of proxy "waits for the result" events,0,milvus,,, -milvus.proxy.sq.wait_result_latency.count,count,,,,Count of proxy "waits for the result" events,0,milvus,,, -milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,Sum of latencies for proxy "waits for the result" events,0,milvus,,, +milvus.proxy.sq.decode_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.decode_result_latency.count,count,,,,"Count of proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.count,count,,,,"Count of proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'waits for the result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.count,count,,,,"Count of proxy 'waits for the result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'waits for the result' events",0,milvus,,, milvus.proxy.sync_segment_request_length.bucket,count,,,,Histogram bucket for the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, milvus.proxy.sync_segment_request_length.count,count,,,,Count of assigning segments for insert events,0,milvus,,, milvus.proxy.sync_segment_request_length.sum,count,,,,Sum of the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, -milvus.proxy.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, milvus.quant.compute_cnt.bucket,count,,,,Histogram bucket for the quant compute count per request,0,milvus,,, milvus.quant.compute_cnt.count,count,,,,Count of requests triggering quant compute events,0,milvus,,, milvus.quant.compute_cnt.sum,count,,,,Sum of quant compute event counts,0,milvus,,, milvus.querycoord.collection_num,gauge,,,,Number of collections,0,milvus,,, -milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,Current target checkpoint timestamp in unix seconds,0,milvus,,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,Current target checkpoint timestamp in Unix seconds,0,milvus,,, milvus.querycoord.load.latency.bucket,count,,,,Histogram bucket Latency of load the entire collection,0,milvus,,, milvus.querycoord.load.latency.count,count,,,,Latency of load the entire collection,0,milvus,,, milvus.querycoord.load.latency.sum,count,,millisecond,,Latency of load the entire collection,0,milvus,,, @@ -244,9 +244,9 @@ milvus.querynode.apply_bf_latency.bucket,count,,,,Histogram bucket for the apply milvus.querynode.apply_bf_latency.count,count,,,,Count of apply brute-force events,0,milvus,,, milvus.querynode.apply_bf_latency.sum,count,,millisecond,,Sum of the apply brute-force cost in ms,0,milvus,,, milvus.querynode.collection_num,gauge,,,,Number of collections loaded,0,milvus,,, -milvus.querynode.consume.bytes_counter.count,count,,,,,0,milvus,,, +milvus.querynode.consume.bytes_counter.count,count,,,,Count of consumed bytes,0,milvus,,, milvus.querynode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, -milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,Number of bytes evicted from disk cache,0,milvus,,, milvus.querynode.disk.cache.evict.count,count,,,,Number of segments evicted from disk cache,0,milvus,,, milvus.querynode.disk.cache.evict.duration.count,count,,,,Total time cost of evicting segments from disk cache,0,milvus,,, @@ -260,10 +260,10 @@ milvus.querynode.disk.cache.load.global_duration.bucket,count,,,,Histogram bucke milvus.querynode.disk.cache.load.global_duration.count,count,,,,Count of loading segments from disk cache events,0,milvus,,, milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,Sum of the global duration of loading segments from disk cache,0,milvus,,, milvus.querynode.disk.used_size,gauge,,mebibyte,,Disk used size(MB),0,milvus,,, -milvus.querynode.dml_vchannel_num,gauge,,,,Number of dmlChannels watched,0,milvus,,, +milvus.querynode.dml_vchannel_num,gauge,,,,Number of data manipulation language channels watched,0,milvus,,, milvus.querynode.entity.num,gauge,,,,"Number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, milvus.querynode.entity.size,gauge,,byte,,"Entities' memory size, clustered by collection and state",0,milvus,,, -milvus.querynode.execute_bytes_counter.count,count,,,,,0,milvus,,, +milvus.querynode.execute_bytes_counter.count,count,,,,Count of execute bytes,0,milvus,,, milvus.querynode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, milvus.querynode.forward_delete_latency.bucket,count,,,,Histogram bucket for the forward delete cost in ms,0,milvus,,, milvus.querynode.forward_delete_latency.count,count,,,,Count of forward delete events,0,milvus,,, @@ -335,9 +335,9 @@ milvus.querynode.sq.wait_tsafe_latency.bucket,count,,,,Histogram bucket for the milvus.querynode.sq.wait_tsafe_latency.count,count,,,,Count of occurrences of search or query waiting for tsafe,0,milvus,,, milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,Sum of the time that search or query wait for tsafe,0,milvus,,, milvus.querynode.wait_processing_msg,gauge,,,,Count of wait processing msg,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Histogram bucket for the latencies of watch dml channel,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.count,count,,,,Count of watch dml channel events,0,milvus,,, -milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Sum of the latencies of watch dml channel events,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Histogram bucket for the latencies of watch data manipulation language channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.count,count,,,,Count of watch data manipulation language channel events,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Sum of the latencies of watch data manipulation language channel events,0,milvus,,, milvus.queue.latency.bucket,count,,,,Histogram bucket for the queue latency per request,0,milvus,,, milvus.queue.latency.count,count,,,,Count of request with queue latency,0,milvus,,, milvus.queue.latency.sum,count,,,,Sum of queue latencies,0,milvus,,, @@ -367,7 +367,7 @@ milvus.rootcoord.indexed_entity_num,gauge,,,,"Indexed number of entities, cluste milvus.rootcoord.msgstream_obj_num,gauge,,,,Number of message streams,0,milvus,,, milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,,, milvus.rootcoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, -milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,Now time minus tt per physical channel,0,milvus,,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, milvus.rootcoord.proxy_num,gauge,,,,Number of proxy nodes managered by rootcoord,0,milvus,,, milvus.rootcoord.qn_mem_high_water_level,gauge,,,,Querynode memory high water level,0,milvus,,, milvus.rootcoord.sync_timetick_latency.bucket,count,,,,Histogram bucket for the latencies of synchronizing timetick message,0,milvus,,, @@ -382,6 +382,6 @@ milvus.search.latency.sum,count,,millisecond,,Sum of search latencies (ms),0,mil milvus.search.topk.bucket,count,,,,Histogram bucket for search topk,0,milvus,,, milvus.search.topk.count,count,,,,Count aggregation of search topk,0,milvus,,, milvus.search.topk.sum,count,,,,Sum aggregation of search topk,0,milvus,,, -milvus.storage.kv_size,gauge,,,,Kv size stats,0,milvus,,, +milvus.storage.kv_size,gauge,,,,Key-value size stats,0,milvus,,, milvus.storage.op.count,count,,,,Count of persistent data operation,0,milvus,,, milvus.storage.request_latency,gauge,,,,Request latency on the client side,0,milvus,,, From 11c337e682d48ef1100c999fac012ea7b38d686c Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 23 Dec 2024 15:05:38 +0200 Subject: [PATCH 29/31] Move metrics out of standalone test --- milvus/tests/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/milvus/tests/common.py b/milvus/tests/common.py index 4b2999f41bee7..e225c01ae90e1 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -199,7 +199,6 @@ def get_fixture_path(filename): 'milvus.querynode.load.segment.latency.sum': 'monotonic_count', 'milvus.querynode.load.segment.latency.count': 'monotonic_count', 'milvus.querynode.load.segment.latency.bucket': 'monotonic_count', - 'milvus.querynode.partition_num': 'gauge', 'milvus.querynode.process_insert_or_delete_latency.sum': 'monotonic_count', 'milvus.querynode.process_insert_or_delete_latency.count': 'monotonic_count', 'milvus.querynode.process_insert_or_delete_latency.bucket': 'monotonic_count', @@ -234,7 +233,6 @@ def get_fixture_path(filename): 'milvus.querynode.segment.latency_per_vector.sum': 'monotonic_count', 'milvus.querynode.segment.latency_per_vector.count': 'monotonic_count', 'milvus.querynode.segment.latency_per_vector.bucket': 'monotonic_count', - 'milvus.querynode.segment.num': 'gauge', 'milvus.querynode.sq.core_latency.sum': 'monotonic_count', 'milvus.querynode.sq.core_latency.count': 'monotonic_count', 'milvus.querynode.sq.core_latency.bucket': 'monotonic_count', @@ -247,7 +245,6 @@ def get_fixture_path(filename): 'milvus.querynode.sq.reduce_latency.sum': 'monotonic_count', 'milvus.querynode.sq.reduce_latency.count': 'monotonic_count', 'milvus.querynode.sq.reduce_latency.bucket': 'monotonic_count', - 'milvus.querynode.sq.req.count': 'monotonic_count', 'milvus.querynode.sq.req.latency.sum': 'monotonic_count', 'milvus.querynode.sq.req.latency.count': 'monotonic_count', 'milvus.querynode.sq.req.latency.bucket': 'monotonic_count', @@ -407,6 +404,9 @@ def get_fixture_path(filename): 'milvus.querynode.entity.num': 'gauge', 'milvus.querynode.entity.size': 'gauge', 'milvus.querynode.msg_dispatcher_tt_lag_ms': 'gauge', + 'milvus.querynode.partition_num': 'gauge', + 'milvus.querynode.segment.num': 'gauge', + 'milvus.querynode.sq.req.count': 'monotonic_count', 'milvus.rootcoord.sync_timetick_latency.bucket': 'count', 'milvus.rootcoord.sync_timetick_latency.count': 'count', 'milvus.rootcoord.timestamp': 'gauge', From c628898aec3dedda0710dec0f96e25ac44290f0a Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 23 Dec 2024 15:19:05 +0200 Subject: [PATCH 30/31] Fix unit tests --- milvus/metadata.csv | 2 +- milvus/tests/common.py | 2 +- milvus/tests/compose/docker-compose.yaml | 2 +- milvus/tests/compose/requirements.txt | 2 +- milvus/tests/fixtures/milvus_payload.txt | 4978 ++++++++++++---------- milvus/tests/test_e2e.py | 5 +- 6 files changed, 2619 insertions(+), 2372 deletions(-) diff --git a/milvus/metadata.csv b/milvus/metadata.csv index 9725cfdb508e5..a845480cf1521 100644 --- a/milvus/metadata.csv +++ b/milvus/metadata.csv @@ -27,7 +27,7 @@ milvus.datacoord.datanode_num,gauge,,,,Number of data nodes,0,milvus,,, milvus.datacoord.import_tasks,gauge,,,,The import tasks grouping by type and state,0,milvus,,, milvus.datacoord.index.node_num,gauge,,,,Number of IndexNodes managed by IndexCoord,0,milvus,,, milvus.datacoord.index.req.count,count,,,,Number of building index requests,0,milvus,,, -milvus.datacoord.index.task_count,gauge,,,,Number of index tasks of each type,0,milvus,,, +milvus.datacoord.index.task,gauge,,,,Number of index tasks of each type,0,milvus,,, milvus.datacoord.segment_num,gauge,,,,Number of segments,0,milvus,,, milvus.datacoord.stored.binlog_size,gauge,,byte,,Binlog size of healthy segments,0,milvus,,, milvus.datacoord.stored.index_files_size,gauge,,byte,,Index files size of the segments,0,milvus,,, diff --git a/milvus/tests/common.py b/milvus/tests/common.py index e225c01ae90e1..e2524182310f2 100644 --- a/milvus/tests/common.py +++ b/milvus/tests/common.py @@ -63,6 +63,7 @@ def get_fixture_path(filename): 'milvus.datacoord.consume_datanode_tt_lag_ms': 'gauge', 'milvus.datacoord.datanode_num': 'gauge', 'milvus.datacoord.import_tasks': 'gauge', + 'milvus.datacoord.index.task': 'gauge', 'milvus.datacoord.index.node_num': 'gauge', 'milvus.datacoord.index.req.count': 'monotonic_count', 'milvus.datacoord.segment_num': 'gauge', @@ -390,7 +391,6 @@ def get_fixture_path(filename): OTHER_TEST_METRICS = { 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', - 'milvus.datacoord.index.task_count': 'gauge', 'milvus.datacoord.stored.binlog_size': 'gauge', 'milvus.datacoord.stored.index_files_size': 'gauge', 'milvus.datacoord.stored.rows_num': 'gauge', diff --git a/milvus/tests/compose/docker-compose.yaml b/milvus/tests/compose/docker-compose.yaml index f7225021924ff..d6a4e03eeaf0d 100644 --- a/milvus/tests/compose/docker-compose.yaml +++ b/milvus/tests/compose/docker-compose.yaml @@ -1,6 +1,6 @@ services: milvus: - image: milvusdb/milvus + image: milvusdb/milvus:v2.5.0 security_opt: - seccomp:unconfined environment: diff --git a/milvus/tests/compose/requirements.txt b/milvus/tests/compose/requirements.txt index 2cee9519b7f56..f96eefaacf1ea 100644 --- a/milvus/tests/compose/requirements.txt +++ b/milvus/tests/compose/requirements.txt @@ -1,2 +1,2 @@ numpy==1.26.4 -pymilvus==2.2.17 +pymilvus==2.5.1 diff --git a/milvus/tests/fixtures/milvus_payload.txt b/milvus/tests/fixtures/milvus_payload.txt index 605f8948ca370..77895e0a21a2d 100644 --- a/milvus/tests/fixtures/milvus_payload.txt +++ b/milvus/tests/fixtures/milvus_payload.txt @@ -1,60 +1,60 @@ # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. # TYPE go_gc_duration_seconds summary -go_gc_duration_seconds{quantile="0"} 1.3251e-05 -go_gc_duration_seconds{quantile="0.25"} 7.625e-05 -go_gc_duration_seconds{quantile="0.5"} 9.6499e-05 -go_gc_duration_seconds{quantile="0.75"} 0.000127667 -go_gc_duration_seconds{quantile="1"} 0.001003166 -go_gc_duration_seconds_sum 0.006592997 -go_gc_duration_seconds_count 33 +go_gc_duration_seconds{quantile="0"} 2.2583e-05 +go_gc_duration_seconds{quantile="0.25"} 5.5624e-05 +go_gc_duration_seconds{quantile="0.5"} 7e-05 +go_gc_duration_seconds{quantile="0.75"} 0.000267583 +go_gc_duration_seconds{quantile="1"} 0.000976792 +go_gc_duration_seconds_sum 0.003638707 +go_gc_duration_seconds_count 18 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge -go_goroutines 360 +go_goroutines 385 # HELP go_info Information about the Go environment. # TYPE go_info gauge -go_info{version="go1.21.11"} 1 +go_info{version="go1.22.0"} 1 # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. # TYPE go_memstats_alloc_bytes gauge -go_memstats_alloc_bytes 6.5780264e+07 +go_memstats_alloc_bytes 8.1514072e+07 # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. # TYPE go_memstats_alloc_bytes_total counter -go_memstats_alloc_bytes_total 1.225491696e+09 +go_memstats_alloc_bytes_total 6.38517176e+08 # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. # TYPE go_memstats_buck_hash_sys_bytes gauge -go_memstats_buck_hash_sys_bytes 1.750476e+06 +go_memstats_buck_hash_sys_bytes 1.803289e+06 # HELP go_memstats_frees_total Total number of frees. # TYPE go_memstats_frees_total counter -go_memstats_frees_total 1.2413451e+07 +go_memstats_frees_total 2.890576e+06 # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. # TYPE go_memstats_gc_sys_bytes gauge -go_memstats_gc_sys_bytes 7.998296e+06 +go_memstats_gc_sys_bytes 5.005784e+06 # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. # TYPE go_memstats_heap_alloc_bytes gauge -go_memstats_heap_alloc_bytes 6.5780264e+07 +go_memstats_heap_alloc_bytes 8.1514072e+07 # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. # TYPE go_memstats_heap_idle_bytes gauge -go_memstats_heap_idle_bytes 7.7365248e+07 +go_memstats_heap_idle_bytes 1.70811392e+08 # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. # TYPE go_memstats_heap_inuse_bytes gauge -go_memstats_heap_inuse_bytes 7.3826304e+07 +go_memstats_heap_inuse_bytes 9.3151232e+07 # HELP go_memstats_heap_objects Number of allocated objects. # TYPE go_memstats_heap_objects gauge -go_memstats_heap_objects 609354 +go_memstats_heap_objects 689980 # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. # TYPE go_memstats_heap_released_bytes gauge -go_memstats_heap_released_bytes 7.02464e+07 +go_memstats_heap_released_bytes 1.550336e+08 # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. # TYPE go_memstats_heap_sys_bytes gauge -go_memstats_heap_sys_bytes 1.51191552e+08 +go_memstats_heap_sys_bytes 2.63962624e+08 # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. # TYPE go_memstats_last_gc_time_seconds gauge -go_memstats_last_gc_time_seconds 1.7325304762589433e+09 +go_memstats_last_gc_time_seconds 1.7349596816399348e+09 # HELP go_memstats_lookups_total Total number of pointer lookups. # TYPE go_memstats_lookups_total counter go_memstats_lookups_total 0 # HELP go_memstats_mallocs_total Total number of mallocs. # TYPE go_memstats_mallocs_total counter -go_memstats_mallocs_total 1.3022805e+07 +go_memstats_mallocs_total 3.580556e+06 # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. # TYPE go_memstats_mcache_inuse_bytes gauge go_memstats_mcache_inuse_bytes 12000 @@ -63,150 +63,150 @@ go_memstats_mcache_inuse_bytes 12000 go_memstats_mcache_sys_bytes 15600 # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. # TYPE go_memstats_mspan_inuse_bytes gauge -go_memstats_mspan_inuse_bytes 1.178184e+06 +go_memstats_mspan_inuse_bytes 1.24096e+06 # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. # TYPE go_memstats_mspan_sys_bytes gauge -go_memstats_mspan_sys_bytes 1.336272e+06 +go_memstats_mspan_sys_bytes 1.51776e+06 # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. # TYPE go_memstats_next_gc_bytes gauge -go_memstats_next_gc_bytes 7.7677144e+07 +go_memstats_next_gc_bytes 1.33652608e+08 # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. # TYPE go_memstats_other_sys_bytes gauge -go_memstats_other_sys_bytes 2.083652e+06 +go_memstats_other_sys_bytes 1.994135e+06 # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. # TYPE go_memstats_stack_inuse_bytes gauge -go_memstats_stack_inuse_bytes 3.997696e+06 +go_memstats_stack_inuse_bytes 4.42368e+06 # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. # TYPE go_memstats_stack_sys_bytes gauge -go_memstats_stack_sys_bytes 3.997696e+06 +go_memstats_stack_sys_bytes 4.42368e+06 # HELP go_memstats_sys_bytes Number of bytes obtained from system. # TYPE go_memstats_sys_bytes gauge -go_memstats_sys_bytes 1.68373544e+08 +go_memstats_sys_bytes 2.78722872e+08 # HELP go_threads Number of OS threads created. # TYPE go_threads gauge -go_threads 18 +go_threads 21 # HELP milvus_build_info Build information of milvus # TYPE milvus_build_info gauge -milvus_build_info{built="Tue Nov 5 07:22:45 UTC 2024",git_commit="b7c80f9",version="v2.4.15"} 1 +milvus_build_info{built="Fri Dec 20 13:51:45 UTC 2024",git_commit="7fe2cd0",version="v2.5.0"} 1 # HELP milvus_cgo_active_future_total Total number of active futures. # TYPE milvus_cgo_active_future_total gauge -milvus_cgo_active_future_total{node_id="2"} 0 +milvus_cgo_active_future_total{node_id="37"} 0 # HELP milvus_cgo_cgo_duration_seconds Histogram of cgo call duration in seconds. # TYPE milvus_cgo_cgo_duration_seconds histogram -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1.0000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="2.5000000000000004e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="5.000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="9.999999999999999e-06"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="1.9999999999999998e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="4.9999999999999996e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="9.999999999999999e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.00025"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.0005"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.001"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.002"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="0.01"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="2",le="+Inf"} 7 -milvus_cgo_cgo_duration_seconds_sum{name="future_destroy",node_id="2"} 1.1126e-05 -milvus_cgo_cgo_duration_seconds_count{name="future_destroy",node_id="2"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1.0000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="2.5000000000000004e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="5.000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="9.999999999999999e-06"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="1.9999999999999998e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="4.9999999999999996e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="9.999999999999999e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.00025"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.0005"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.001"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.002"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="0.01"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="2",le="+Inf"} 7 -milvus_cgo_cgo_duration_seconds_sum{name="future_go_register_ready_callback",node_id="2"} 1.5081999999999999e-05 -milvus_cgo_cgo_duration_seconds_count{name="future_go_register_ready_callback",node_id="2"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1.0000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="2.5000000000000004e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="5.000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="9.999999999999999e-06"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="1.9999999999999998e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="4.9999999999999996e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="9.999999999999999e-05"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.00025"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.0005"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.001"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.002"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="0.01"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="2",le="+Inf"} 7 -milvus_cgo_cgo_duration_seconds_sum{name="future_leak_and_get",node_id="2"} 2.5625e-05 -milvus_cgo_cgo_duration_seconds_count{name="future_leak_and_get",node_id="2"} 7 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1.0000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="2.5000000000000004e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="5.000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="9.999999999999999e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="1.9999999999999998e-05"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="4.9999999999999996e-05"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="9.999999999999999e-05"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.00025"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.0005"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.001"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.002"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="0.01"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="2",le="+Inf"} 5 -milvus_cgo_cgo_duration_seconds_sum{name="retrieve",node_id="2"} 0.000289917 -milvus_cgo_cgo_duration_seconds_count{name="retrieve",node_id="2"} 5 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1.0000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="2.5000000000000004e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="5.000000000000001e-07"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="9.999999999999999e-06"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="1.9999999999999998e-05"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="4.9999999999999996e-05"} 0 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="9.999999999999999e-05"} 1 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.00025"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.0005"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.001"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.002"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="0.01"} 2 -milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="2",le="+Inf"} 2 -milvus_cgo_cgo_duration_seconds_sum{name="search",node_id="2"} 0.000204042 -milvus_cgo_cgo_duration_seconds_count{name="search",node_id="2"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1.9999999999999998e-05"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_destroy",node_id="37"} 0.000116711 +milvus_cgo_cgo_duration_seconds_count{name="future_destroy",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1.9999999999999998e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_go_register_ready_callback",node_id="37"} 6.487499999999999e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_go_register_ready_callback",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1.9999999999999998e-05"} 6 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_leak_and_get",node_id="37"} 9.270699999999999e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_leak_and_get",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="4.9999999999999996e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="9.999999999999999e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.00025"} 1 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.0005"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.001"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.002"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.01"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="+Inf"} 5 +milvus_cgo_cgo_duration_seconds_sum{name="retrieve",node_id="37"} 0.0015534569999999998 +milvus_cgo_cgo_duration_seconds_count{name="retrieve",node_id="37"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="4.9999999999999996e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="9.999999999999999e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.00025"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.0005"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.001"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.002"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.01"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="+Inf"} 2 +milvus_cgo_cgo_duration_seconds_sum{name="search",node_id="37"} 0.000372251 +milvus_cgo_cgo_duration_seconds_count{name="search",node_id="37"} 2 # HELP milvus_cgo_cgo_queue_duration_seconds Duration of cgo call in queue. # TYPE milvus_cgo_cgo_queue_duration_seconds histogram -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1e-08"} 0 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1.0000000000000001e-07"} 14 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="2.5000000000000004e-07"} 21 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="5.000000000000001e-07"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1e-06"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="9.999999999999999e-06"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="1.9999999999999998e-05"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="4.9999999999999996e-05"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="9.999999999999999e-05"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.00025"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.0005"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.001"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.002"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="0.01"} 28 -milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="2",le="+Inf"} 28 -milvus_cgo_cgo_queue_duration_seconds_sum{node_id="2"} 5.0410000000000005e-06 -milvus_cgo_cgo_queue_duration_seconds_count{node_id="2"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1.0000000000000001e-07"} 1 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="2.5000000000000004e-07"} 13 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="5.000000000000001e-07"} 18 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1e-06"} 22 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="9.999999999999999e-06"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1.9999999999999998e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="4.9999999999999996e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="9.999999999999999e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.00025"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.0005"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.001"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.002"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.01"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="+Inf"} 28 +milvus_cgo_cgo_queue_duration_seconds_sum{node_id="37"} 1.6873e-05 +milvus_cgo_cgo_queue_duration_seconds_count{node_id="37"} 28 # HELP milvus_cgo_running_cgo_call_total Total number of running cgo calls. # TYPE milvus_cgo_running_cgo_call_total gauge -milvus_cgo_running_cgo_call_total{node_id="2"} 0 +milvus_cgo_running_cgo_call_total{node_id="37"} 0 # HELP milvus_datacoord_collection_num number of collections # TYPE milvus_datacoord_collection_num gauge milvus_datacoord_collection_num 0 # HELP milvus_datacoord_consume_datanode_tt_lag_ms now time minus tt per physical channel # TYPE milvus_datacoord_consume_datanode_tt_lag_ms gauge -milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0",node_id="2"} 243 -milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0_454172268311087164v0",node_id="2"} 243 +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10",node_id="37"} 487 +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10_454809251903439155v0",node_id="37"} 487 # HELP milvus_datacoord_datanode_num number of data nodes # TYPE milvus_datacoord_datanode_num gauge milvus_datacoord_datanode_num 1 @@ -227,21 +227,133 @@ milvus_datacoord_index_node_num 1 # TYPE milvus_datacoord_index_req_count counter milvus_datacoord_index_req_count{status="success"} 1 milvus_datacoord_index_req_count{status="total"} 1 +# HELP milvus_datacoord_index_task_count number of index tasks of each type +# TYPE milvus_datacoord_index_task_count gauge +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="unissued"} 0 # HELP milvus_datacoord_segment_num number of segments # TYPE milvus_datacoord_segment_num gauge -milvus_datacoord_segment_num{segment_level="L0",segment_state="Dropped"} 1 -milvus_datacoord_segment_num{segment_level="L0",segment_state="Flushed"} 0 -milvus_datacoord_segment_num{segment_level="L1",segment_state="Dropped"} 1 -milvus_datacoord_segment_num{segment_level="L1",segment_state="Flushed"} 0 -milvus_datacoord_segment_num{segment_level="L1",segment_state="Flushing"} 0 -milvus_datacoord_segment_num{segment_level="L1",segment_state="Growing"} 0 -milvus_datacoord_segment_num{segment_level="L1",segment_state="Sealed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="sorted",segment_level="L1",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="sorted",segment_level="L1",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L0",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L0",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Flushing"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Growing"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Sealed"} 0 +# HELP milvus_datacoord_task_count number of index tasks of each type +# TYPE milvus_datacoord_task_count gauge +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 # HELP milvus_datacoord_task_execute_max_latency latency of task execute operation # TYPE milvus_datacoord_task_execute_max_latency histogram milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1"} 0 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="100"} 0 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="500"} 0 -milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1000"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1000"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5000"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="10000"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="20000"} 1 @@ -254,9 +366,27 @@ milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="J milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5e+06"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1e+07"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="+Inf"} 1 -milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeIndexJob"} 1540 +milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeIndexJob"} 971 milvus_datacoord_task_execute_max_latency_count{status="executing",task_type="JobTypeIndexJob"} 1 -milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="100"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="500"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeStatsJob"} 533 +milvus_datacoord_task_execute_max_latency_count{status="executing",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1"} 0 milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="100"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="500"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1000"} 1 @@ -272,1279 +402,1278 @@ milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="Job milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="5e+06"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1e+07"} 1 milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="+Inf"} 1 -milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeIndexJob"} 15 milvus_datacoord_task_execute_max_latency_count{status="pending",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="100"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="500"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeStatsJob"} 5 +milvus_datacoord_task_execute_max_latency_count{status="pending",task_type="JobTypeStatsJob"} 1 # HELP milvus_datanode_autoflush_buffer_op_count count of auto flush buffer operations # TYPE milvus_datanode_autoflush_buffer_op_count counter -milvus_datanode_autoflush_buffer_op_count{node_id="2",segment_level="L1",status="success"} 1 +milvus_datanode_autoflush_buffer_op_count{node_id="37",segment_level="L1",status="success"} 1 # HELP milvus_datanode_consume_bytes_count # TYPE milvus_datanode_consume_bytes_count counter -milvus_datanode_consume_bytes_count{msg_type="delete",node_id="2"} 231 -milvus_datanode_consume_bytes_count{msg_type="insert",node_id="2"} 191554 +milvus_datanode_consume_bytes_count{msg_type="delete",node_id="37"} 241 +milvus_datanode_consume_bytes_count{msg_type="insert",node_id="37"} 191574 # HELP milvus_datanode_encode_buffer_latency latency of encode buffer data # TYPE milvus_datanode_encode_buffer_latency histogram -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="5"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="10"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="20"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="50"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="100"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="200"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="300"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="400"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="500"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="600"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="700"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="800"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="900"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="1000"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="1500"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="3000"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="10000"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="60000"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L0",le="+Inf"} 1 -milvus_datanode_encode_buffer_latency_sum{node_id="2",segment_level="L0"} 4 -milvus_datanode_encode_buffer_latency_count{node_id="2",segment_level="L0"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="5"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="10"} 1 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="20"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="50"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="100"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="200"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="300"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="400"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="500"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="600"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="700"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="800"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="900"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="1000"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="1500"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="3000"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="10000"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="60000"} 2 -milvus_datanode_encode_buffer_latency_bucket{node_id="2",segment_level="L1",le="+Inf"} 2 -milvus_datanode_encode_buffer_latency_sum{node_id="2",segment_level="L1"} 16 -milvus_datanode_encode_buffer_latency_count{node_id="2",segment_level="L1"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="1"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="2"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="4"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="8"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="16"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="32"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="64"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="128"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="256"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="512"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="1024"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="2048"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="4096"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="8192"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="16384"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="32768"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="65536"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="131072"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="+Inf"} 1 +milvus_datanode_encode_buffer_latency_sum{node_id="37",segment_level="L0"} 6 +milvus_datanode_encode_buffer_latency_count{node_id="37",segment_level="L0"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="1"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="2"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="4"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="8"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="16"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="32"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="64"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="128"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="256"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="512"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="1024"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="2048"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="4096"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="8192"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="16384"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="32768"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="65536"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="131072"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="+Inf"} 2 +milvus_datanode_encode_buffer_latency_sum{node_id="37",segment_level="L1"} 20 +milvus_datanode_encode_buffer_latency_count{node_id="37",segment_level="L1"} 2 # HELP milvus_datanode_flowgraph_num number of flowgraphs # TYPE milvus_datanode_flowgraph_num gauge -milvus_datanode_flowgraph_num{node_id="2"} 0 +milvus_datanode_flowgraph_num{node_id="37"} 0 # HELP milvus_datanode_flush_buffer_op_count count of flush buffer operations # TYPE milvus_datanode_flush_buffer_op_count counter -milvus_datanode_flush_buffer_op_count{node_id="2",segment_level="L0",status="success"} 1 -milvus_datanode_flush_buffer_op_count{node_id="2",segment_level="L1",status="success"} 2 +milvus_datanode_flush_buffer_op_count{node_id="37",segment_level="L0",status="success"} 1 +milvus_datanode_flush_buffer_op_count{node_id="37",segment_level="L1",status="success"} 2 # HELP milvus_datanode_flush_req_count count of flush request # TYPE milvus_datanode_flush_req_count counter -milvus_datanode_flush_req_count{node_id="2",status="success"} 1 -milvus_datanode_flush_req_count{node_id="2",status="total"} 1 +milvus_datanode_flush_req_count{node_id="37",status="success"} 1 +milvus_datanode_flush_req_count{node_id="37",status="total"} 3 # HELP milvus_datanode_flushed_data_rows num of rows flushed to storage # TYPE milvus_datanode_flushed_data_rows counter -milvus_datanode_flushed_data_rows{data_source="streaming",node_id="2"} 3001 +milvus_datanode_flushed_data_rows{data_source="streaming",node_id="37"} 3001 # HELP milvus_datanode_flushed_data_size byte size of data flushed to storage # TYPE milvus_datanode_flushed_data_size counter -milvus_datanode_flushed_data_size{data_source="streaming",node_id="2",segment_level="L0"} 616 -milvus_datanode_flushed_data_size{data_source="streaming",node_id="2",segment_level="L1"} 226975 +milvus_datanode_flushed_data_size{data_source="streaming",node_id="37",segment_level="L0"} 620 +milvus_datanode_flushed_data_size{data_source="streaming",node_id="37",segment_level="L1"} 226983 # HELP milvus_datanode_msg_rows_count count of rows consumed from msgStream # TYPE milvus_datanode_msg_rows_count counter -milvus_datanode_msg_rows_count{msg_type="delete",node_id="2"} 2 -milvus_datanode_msg_rows_count{msg_type="insert",node_id="2"} 3001 +milvus_datanode_msg_rows_count{msg_type="delete",node_id="37"} 2 +milvus_datanode_msg_rows_count{msg_type="insert",node_id="37"} 3001 # HELP milvus_datanode_save_latency latency of saving flush data to storage # TYPE milvus_datanode_save_latency histogram -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="0"} 0 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="10"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="100"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="200"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="400"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="1000"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="10000"} 1 -milvus_datanode_save_latency_bucket{msg_type="L0",node_id="2",le="+Inf"} 1 -milvus_datanode_save_latency_sum{msg_type="L0",node_id="2"} 3 -milvus_datanode_save_latency_count{msg_type="L0",node_id="2"} 1 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="0"} 0 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="10"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="100"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="200"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="400"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="1000"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="10000"} 2 -milvus_datanode_save_latency_bucket{msg_type="L1",node_id="2",le="+Inf"} 2 -milvus_datanode_save_latency_sum{msg_type="L1",node_id="2"} 8 -milvus_datanode_save_latency_count{msg_type="L1",node_id="2"} 2 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="10"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="100"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="200"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="400"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="1000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="10000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="+Inf"} 1 +milvus_datanode_save_latency_sum{msg_type="L0",node_id="37"} 5 +milvus_datanode_save_latency_count{msg_type="L0",node_id="37"} 1 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="10"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="100"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="200"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="400"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="1000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="10000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="+Inf"} 2 +milvus_datanode_save_latency_sum{msg_type="L1",node_id="37"} 14 +milvus_datanode_save_latency_count{msg_type="L1",node_id="37"} 2 # HELP milvus_flushed_segment_file_num the num of files for flushed segment # TYPE milvus_flushed_segment_file_num histogram -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="5"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="10"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="20"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="50"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="100"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="200"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="300"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="400"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="600"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="700"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="800"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="900"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="3000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="10000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="2"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="4"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="131072"} 1 milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="+Inf"} 1 milvus_flushed_segment_file_num_sum{segment_file_type="delete_file"} 0 milvus_flushed_segment_file_num_count{segment_file_type="delete_file"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="5"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="10"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="20"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="50"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="100"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="200"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="300"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="400"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="600"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="700"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="800"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="900"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="3000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="10000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="60000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="+Inf"} 1 -milvus_flushed_segment_file_num_sum{segment_file_type="index_file"} 1 -milvus_flushed_segment_file_num_count{segment_file_type="index_file"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="5"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="10"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="20"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="50"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="100"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="200"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="300"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="400"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="600"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="700"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="800"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="900"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="3000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="10000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="2"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="4"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="8"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="16"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="32"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="64"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="128"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="256"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="512"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1024"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="2048"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="4096"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="8192"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="16384"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="32768"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="65536"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="131072"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="+Inf"} 11 +milvus_flushed_segment_file_num_sum{segment_file_type="index_file"} 11 +milvus_flushed_segment_file_num_count{segment_file_type="index_file"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="2"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="4"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="131072"} 1 milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="+Inf"} 1 milvus_flushed_segment_file_num_sum{segment_file_type="insert_file"} 5 milvus_flushed_segment_file_num_count{segment_file_type="insert_file"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="5"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="10"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="20"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="50"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="100"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="200"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="300"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="400"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="600"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="700"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="800"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="900"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1500"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="3000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="10000"} 1 -milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="60000"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="2"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="4"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="131072"} 1 milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="+Inf"} 1 milvus_flushed_segment_file_num_sum{segment_file_type="stat_file"} 2 milvus_flushed_segment_file_num_count{segment_file_type="stat_file"} 1 # HELP milvus_indexnode_build_index_latency latency of build index for segment # TYPE milvus_indexnode_build_index_latency histogram -milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.001"} 0 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.1"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="0.5"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="1"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="5"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="10"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="20"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="50"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="100"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="250"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="500"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="1000"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="3600"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="5000"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="10000"} 1 -milvus_indexnode_build_index_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_indexnode_build_index_latency_sum{node_id="2"} 0.026148791 -milvus_indexnode_build_index_latency_count{node_id="2"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_build_index_latency_sum{node_id="37"} 0.033203625 +milvus_indexnode_build_index_latency_count{node_id="37"} 1 # HELP milvus_indexnode_encode_index_latency latency of encoding the index file # TYPE milvus_indexnode_encode_index_latency histogram -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.001"} 0 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.1"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="0.5"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="1"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="5"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="10"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="20"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="50"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="100"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="250"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="500"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="1000"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="3600"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="5000"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="10000"} 1 -milvus_indexnode_encode_index_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_indexnode_encode_index_latency_sum{node_id="2"} 0.00681125 -milvus_indexnode_encode_index_latency_count{node_id="2"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_encode_index_latency_sum{node_id="37"} 0.007635709 +milvus_indexnode_encode_index_latency_count{node_id="37"} 1 # HELP milvus_indexnode_index_task_count number of tasks that index node received # TYPE milvus_indexnode_index_task_count counter -milvus_indexnode_index_task_count{node_id="2",status="success"} 1 +milvus_indexnode_index_task_count{node_id="37",status="success"} 1 # HELP milvus_indexnode_index_task_latency_in_queue latency of index task in queue # TYPE milvus_indexnode_index_task_latency_in_queue histogram -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="5"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="10"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="20"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="50"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="100"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="200"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="300"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="400"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="500"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="600"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="700"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="800"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="900"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="1000"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="1500"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="3000"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="10000"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="60000"} 1 -milvus_indexnode_index_task_latency_in_queue_bucket{node_id="2",le="+Inf"} 1 -milvus_indexnode_index_task_latency_in_queue_sum{node_id="2"} 0 -milvus_indexnode_index_task_latency_in_queue_count{node_id="2"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="1"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="2"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="4"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="8"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="16"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="32"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="64"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="128"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="256"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="512"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="1024"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="2048"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="4096"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="8192"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="16384"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="32768"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="65536"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="131072"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_index_task_latency_in_queue_sum{node_id="37"} 0 +milvus_indexnode_index_task_latency_in_queue_count{node_id="37"} 1 # HELP milvus_indexnode_knowhere_build_index_latency latency of building the index by knowhere # TYPE milvus_indexnode_knowhere_build_index_latency histogram -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.001"} 0 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.1"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="0.5"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="1"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="5"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="10"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="20"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="50"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="100"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="250"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="500"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="1000"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="3600"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="5000"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="10000"} 1 -milvus_indexnode_knowhere_build_index_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_indexnode_knowhere_build_index_latency_sum{node_id="2"} 0.019188333 -milvus_indexnode_knowhere_build_index_latency_count{node_id="2"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_knowhere_build_index_latency_sum{node_id="37"} 0.025299416 +milvus_indexnode_knowhere_build_index_latency_count{node_id="37"} 1 # HELP milvus_indexnode_save_index_latency latency of saving the index file # TYPE milvus_indexnode_save_index_latency histogram -milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.001"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.1"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="0.5"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="1"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="5"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="10"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="20"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="50"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="100"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="250"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="500"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="1000"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="3600"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="5000"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="10000"} 1 -milvus_indexnode_save_index_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_indexnode_save_index_latency_sum{node_id="2"} 4.6958e-05 -milvus_indexnode_save_index_latency_count{node_id="2"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.001"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_save_index_latency_sum{node_id="37"} 6.0041e-05 +milvus_indexnode_save_index_latency_count{node_id="37"} 1 # HELP milvus_meta_kv_size kv size stats # TYPE milvus_meta_kv_size histogram -milvus_meta_kv_size_bucket{meta_op_type="get",le="5"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="10"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="20"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="50"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="100"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="200"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="300"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="400"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="500"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="600"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="700"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="800"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="900"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="1000"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="1500"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="3000"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="10000"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="60000"} 77 -milvus_meta_kv_size_bucket{meta_op_type="get",le="+Inf"} 77 -milvus_meta_kv_size_sum{meta_op_type="get"} -20 -milvus_meta_kv_size_count{meta_op_type="get"} 77 -milvus_meta_kv_size_bucket{meta_op_type="put",le="5"} 1 -milvus_meta_kv_size_bucket{meta_op_type="put",le="10"} 273 -milvus_meta_kv_size_bucket{meta_op_type="put",le="20"} 274 -milvus_meta_kv_size_bucket{meta_op_type="put",le="50"} 282 -milvus_meta_kv_size_bucket{meta_op_type="put",le="100"} 290 -milvus_meta_kv_size_bucket{meta_op_type="put",le="200"} 293 -milvus_meta_kv_size_bucket{meta_op_type="put",le="300"} 293 -milvus_meta_kv_size_bucket{meta_op_type="put",le="400"} 299 -milvus_meta_kv_size_bucket{meta_op_type="put",le="500"} 302 -milvus_meta_kv_size_bucket{meta_op_type="put",le="600"} 302 -milvus_meta_kv_size_bucket{meta_op_type="put",le="700"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="800"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="900"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="1000"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="1500"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="3000"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="10000"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="60000"} 305 -milvus_meta_kv_size_bucket{meta_op_type="put",le="+Inf"} 305 -milvus_meta_kv_size_sum{meta_op_type="put"} 8855 -milvus_meta_kv_size_count{meta_op_type="put"} 305 +milvus_meta_kv_size_bucket{meta_op_type="get",le="1"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="2"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="4"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="8"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="16"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="32"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="64"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="128"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="256"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="512"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="1024"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="2048"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="4096"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="8192"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="16384"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="32768"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="65536"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="131072"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="+Inf"} 89 +milvus_meta_kv_size_sum{meta_op_type="get"} -313 +milvus_meta_kv_size_count{meta_op_type="get"} 89 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="2"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="4"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="8"} 39 +milvus_meta_kv_size_bucket{meta_op_type="put",le="16"} 40 +milvus_meta_kv_size_bucket{meta_op_type="put",le="32"} 40 +milvus_meta_kv_size_bucket{meta_op_type="put",le="64"} 50 +milvus_meta_kv_size_bucket{meta_op_type="put",le="128"} 60 +milvus_meta_kv_size_bucket{meta_op_type="put",le="256"} 63 +milvus_meta_kv_size_bucket{meta_op_type="put",le="512"} 72 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1024"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="2048"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="4096"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="8192"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="16384"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="32768"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="65536"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="131072"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="+Inf"} 76 +milvus_meta_kv_size_sum{meta_op_type="put"} 8526 +milvus_meta_kv_size_count{meta_op_type="put"} 76 # HELP milvus_meta_op_count count of meta operation # TYPE milvus_meta_op_count counter -milvus_meta_op_count{meta_op_type="get",status="success"} 52 -milvus_meta_op_count{meta_op_type="get",status="total"} 52 -milvus_meta_op_count{meta_op_type="put",status="success"} 280 -milvus_meta_op_count{meta_op_type="put",status="total"} 280 +milvus_meta_op_count{meta_op_type="get",status="success"} 63 +milvus_meta_op_count{meta_op_type="get",status="total"} 63 +milvus_meta_op_count{meta_op_type="put",status="success"} 50 +milvus_meta_op_count{meta_op_type="put",status="total"} 50 milvus_meta_op_count{meta_op_type="remove",status="success"} 6 milvus_meta_op_count{meta_op_type="remove",status="total"} 6 -milvus_meta_op_count{meta_op_type="txn",status="success"} 25 -milvus_meta_op_count{meta_op_type="txn",status="total"} 25 +milvus_meta_op_count{meta_op_type="txn",status="success"} 26 +milvus_meta_op_count{meta_op_type="txn",status="total"} 26 # HELP milvus_meta_request_latency request latency on the client side # TYPE milvus_meta_request_latency histogram -milvus_meta_request_latency_bucket{meta_op_type="get",le="5"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="10"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="20"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="50"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="100"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="200"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="300"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="400"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="500"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="600"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="700"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="800"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="900"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="1000"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="1500"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="3000"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="10000"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="60000"} 52 -milvus_meta_request_latency_bucket{meta_op_type="get",le="+Inf"} 52 -milvus_meta_request_latency_sum{meta_op_type="get"} 0 -milvus_meta_request_latency_count{meta_op_type="get"} 52 -milvus_meta_request_latency_bucket{meta_op_type="put",le="5"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="10"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="20"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="50"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="100"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="200"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="300"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="400"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="500"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="600"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="700"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="800"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="900"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="1000"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="1500"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="3000"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="10000"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="60000"} 280 -milvus_meta_request_latency_bucket{meta_op_type="put",le="+Inf"} 280 -milvus_meta_request_latency_sum{meta_op_type="put"} 16 -milvus_meta_request_latency_count{meta_op_type="put"} 280 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="5"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="10"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="20"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="50"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="100"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="200"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="300"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="400"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="500"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="600"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="700"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="800"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="900"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="1000"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="1500"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="3000"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="10000"} 6 -milvus_meta_request_latency_bucket{meta_op_type="remove",le="60000"} 6 +milvus_meta_request_latency_bucket{meta_op_type="get",le="1"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="2"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="4"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="8"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="16"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="32"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="64"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="128"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="256"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="512"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="1024"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="2048"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="4096"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="8192"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="16384"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="32768"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="65536"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="131072"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="+Inf"} 63 +milvus_meta_request_latency_sum{meta_op_type="get"} 7 +milvus_meta_request_latency_count{meta_op_type="get"} 63 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1"} 19 +milvus_meta_request_latency_bucket{meta_op_type="put",le="2"} 35 +milvus_meta_request_latency_bucket{meta_op_type="put",le="4"} 45 +milvus_meta_request_latency_bucket{meta_op_type="put",le="8"} 49 +milvus_meta_request_latency_bucket{meta_op_type="put",le="16"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="32"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="64"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="128"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="256"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="512"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1024"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="2048"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="4096"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="8192"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="16384"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="32768"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="65536"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="131072"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="+Inf"} 50 +milvus_meta_request_latency_sum{meta_op_type="put"} 106 +milvus_meta_request_latency_count{meta_op_type="put"} 50 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="2"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="4"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="8"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="16"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="32"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="64"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="128"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="256"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="512"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1024"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="2048"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="4096"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="8192"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="16384"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="32768"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="65536"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="131072"} 6 milvus_meta_request_latency_bucket{meta_op_type="remove",le="+Inf"} 6 milvus_meta_request_latency_sum{meta_op_type="remove"} 0 milvus_meta_request_latency_count{meta_op_type="remove"} 6 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="5"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="10"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="20"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="50"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="100"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="200"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="300"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="400"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="500"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="600"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="700"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="800"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="900"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="1000"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="1500"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="3000"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="10000"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="60000"} 25 -milvus_meta_request_latency_bucket{meta_op_type="txn",le="+Inf"} 25 -milvus_meta_request_latency_sum{meta_op_type="txn"} 1 -milvus_meta_request_latency_count{meta_op_type="txn"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1"} 22 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="2"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="4"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="8"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="16"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="32"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="64"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="128"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="256"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="512"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1024"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="2048"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="4096"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="8192"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="16384"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="32768"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="65536"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="131072"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="+Inf"} 26 +milvus_meta_request_latency_sum{meta_op_type="txn"} 14 +milvus_meta_request_latency_count{meta_op_type="txn"} 26 # HELP milvus_msg_queue_consumer_num number of consumers # TYPE milvus_msg_queue_consumer_num gauge -milvus_msg_queue_consumer_num{node_id="2",role_name="standalone"} 0 +milvus_msg_queue_consumer_num{node_id="37",role_name="standalone"} 0 # HELP milvus_msgstream_op_count count of stream message operation # TYPE milvus_msgstream_op_count counter milvus_msgstream_op_count{message_op_type="create_consumer",status="success"} 2 milvus_msgstream_op_count{message_op_type="create_consumer",status="total"} 2 milvus_msgstream_op_count{message_op_type="create_producer",status="success"} 18 milvus_msgstream_op_count{message_op_type="create_producer",status="total"} 18 -milvus_msgstream_op_count{message_op_type="produce",status="success"} 59 -milvus_msgstream_op_count{message_op_type="produce",status="total"} 59 +milvus_msgstream_op_count{message_op_type="produce",status="success"} 3594 +milvus_msgstream_op_count{message_op_type="produce",status="total"} 3594 # HELP milvus_msgstream_request_latency request latency on the client side # TYPE milvus_msgstream_request_latency histogram -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="5"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="10"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="20"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="50"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="100"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="200"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="300"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="400"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="500"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="600"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="700"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="800"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="900"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1000"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1500"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="3000"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="10000"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="60000"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="2"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="4"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="8"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="16"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="32"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="64"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="128"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="256"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="512"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1024"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="2048"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="4096"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="8192"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="16384"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="32768"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="65536"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="131072"} 2 milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="+Inf"} 2 milvus_msgstream_request_latency_sum{message_op_type="create_consumer"} 0 milvus_msgstream_request_latency_count{message_op_type="create_consumer"} 2 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="5"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="10"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="20"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="50"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="100"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="200"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="300"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="400"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="500"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="600"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="700"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="800"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="900"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1000"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1500"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="3000"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="10000"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="60000"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="2"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="4"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="8"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="16"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="32"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="64"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="128"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="256"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="512"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1024"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="2048"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="4096"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="8192"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="16384"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="32768"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="65536"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="131072"} 18 milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="+Inf"} 18 milvus_msgstream_request_latency_sum{message_op_type="create_producer"} 0 milvus_msgstream_request_latency_count{message_op_type="create_producer"} 18 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="5"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="10"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="20"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="50"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="100"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="200"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="300"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="400"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="500"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="600"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="700"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="800"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="900"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1000"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1500"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="3000"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="10000"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="60000"} 59 -milvus_msgstream_request_latency_bucket{message_op_type="produce",le="+Inf"} 59 -milvus_msgstream_request_latency_sum{message_op_type="produce"} 0 -milvus_msgstream_request_latency_count{message_op_type="produce"} 59 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1"} 781 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="2"} 1620 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="4"} 2887 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="8"} 3438 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="16"} 3583 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="32"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="64"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="128"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="256"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="512"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1024"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="2048"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="4096"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="8192"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="16384"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="32768"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="65536"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="131072"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="+Inf"} 3594 +milvus_msgstream_request_latency_sum{message_op_type="produce"} 11317 +milvus_msgstream_request_latency_count{message_op_type="produce"} 3594 # HELP milvus_num_node number of nodes and coordinates # TYPE milvus_num_node gauge -milvus_num_node{node_id="2",role_name="datacoord"} 1 -milvus_num_node{node_id="2",role_name="datanode"} 1 -milvus_num_node{node_id="2",role_name="indexnode"} 1 -milvus_num_node{node_id="2",role_name="proxy"} 1 -milvus_num_node{node_id="2",role_name="querycoord"} 1 -milvus_num_node{node_id="2",role_name="querynode"} 1 -milvus_num_node{node_id="2",role_name="rootcoord"} 1 +milvus_num_node{node_id="37",role_name="datacoord"} 1 +milvus_num_node{node_id="37",role_name="datanode"} 1 +milvus_num_node{node_id="37",role_name="indexnode"} 1 +milvus_num_node{node_id="37",role_name="proxy"} 1 +milvus_num_node{node_id="37",role_name="querycoord"} 1 +milvus_num_node{node_id="37",role_name="querynode"} 1 +milvus_num_node{node_id="37",role_name="rootcoord"} 1 # HELP milvus_proxy_apply_pk_latency latency that apply primary key # TYPE milvus_proxy_apply_pk_latency histogram -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="5"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="10"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="20"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="50"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="100"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="200"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="300"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="400"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="500"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="600"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="700"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="800"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="900"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="1000"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="1500"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="3000"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="10000"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="60000"} 2 -milvus_proxy_apply_pk_latency_bucket{node_id="2",le="+Inf"} 2 -milvus_proxy_apply_pk_latency_sum{node_id="2"} 0 -milvus_proxy_apply_pk_latency_count{node_id="2"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="1"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="2"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="4"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="8"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="16"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="32"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="64"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="128"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="256"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="512"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="1024"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="2048"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="4096"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="8192"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="16384"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="32768"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="65536"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="131072"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_apply_pk_latency_sum{node_id="37"} 0 +milvus_proxy_apply_pk_latency_count{node_id="37"} 2 # HELP milvus_proxy_apply_timestamp_latency latency that proxy apply timestamp # TYPE milvus_proxy_apply_timestamp_latency histogram -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="5"} 8041 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="10"} 8042 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="20"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="50"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="100"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="200"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="300"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="400"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="500"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="600"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="700"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="800"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="900"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="1000"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="1500"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="3000"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="10000"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="60000"} 8043 -milvus_proxy_apply_timestamp_latency_bucket{node_id="2",le="+Inf"} 8043 -milvus_proxy_apply_timestamp_latency_sum{node_id="2"} 42 -milvus_proxy_apply_timestamp_latency_count{node_id="2"} 8043 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="1"} 357 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="2"} 523 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="4"} 561 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="8"} 577 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="16"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="32"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="64"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="128"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="256"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="512"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="1024"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="2048"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="4096"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="8192"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="16384"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="32768"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="65536"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="131072"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="+Inf"} 578 +milvus_proxy_apply_timestamp_latency_sum{node_id="37"} 750 +milvus_proxy_apply_timestamp_latency_count{node_id="37"} 578 # HELP milvus_proxy_assign_segmentID_latency latency that proxy get segmentID from dataCoord # TYPE milvus_proxy_assign_segmentID_latency histogram -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="5"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="10"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="20"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="50"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="100"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="200"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="300"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="400"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="500"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="600"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="700"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="800"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="900"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="1000"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="1500"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="3000"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="10000"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="60000"} 2 -milvus_proxy_assign_segmentID_latency_bucket{node_id="2",le="+Inf"} 2 -milvus_proxy_assign_segmentID_latency_sum{node_id="2"} 1 -milvus_proxy_assign_segmentID_latency_count{node_id="2"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="1"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="2"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="4"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="8"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="16"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="32"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="64"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="128"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="256"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="512"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="1024"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="2048"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="4096"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="8192"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="16384"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="32768"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="65536"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="131072"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_assign_segmentID_latency_sum{node_id="37"} 1 +milvus_proxy_assign_segmentID_latency_count{node_id="37"} 2 # HELP milvus_proxy_cache_hit_count count of cache hits/miss # TYPE milvus_proxy_cache_hit_count counter -milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="hit",node_id="2"} 54 -milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="miss",node_id="2"} 3 -milvus_proxy_cache_hit_count{cache_name="GetCollectionInfo",cache_state="hit",node_id="2"} 15 -milvus_proxy_cache_hit_count{cache_name="GetCollectionSchema",cache_state="hit",node_id="2"} 25 -milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="hit",node_id="2"} 6 -milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="miss",node_id="2"} 1 +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="hit",node_id="37"} 51 +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="miss",node_id="37"} 3 +milvus_proxy_cache_hit_count{cache_name="GetCollectionInfo",cache_state="hit",node_id="37"} 19 +milvus_proxy_cache_hit_count{cache_name="GetCollectionSchema",cache_state="hit",node_id="37"} 27 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="hit",node_id="37"} 6 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="miss",node_id="37"} 2 # HELP milvus_proxy_cache_update_latency latency that proxy update cache when cache miss # TYPE milvus_proxy_cache_update_latency histogram -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="5"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="10"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="20"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="50"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="100"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="200"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="300"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="400"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="500"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="600"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="700"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="800"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="900"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="1000"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="1500"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="3000"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="10000"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="60000"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="2",le="+Inf"} 2 -milvus_proxy_cache_update_latency_sum{cache_name="GetCollectionID",node_id="2"} 1 -milvus_proxy_cache_update_latency_count{cache_name="GetCollectionID",node_id="2"} 2 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="5"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="10"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="20"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="50"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="100"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="200"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="300"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="400"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="500"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="600"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="700"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="800"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="900"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="1000"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="1500"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="3000"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="10000"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="60000"} 1 -milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="2",le="+Inf"} 1 -milvus_proxy_cache_update_latency_sum{cache_name="GetShards",node_id="2"} 0 -milvus_proxy_cache_update_latency_count{cache_name="GetShards",node_id="2"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="1"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="2"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="4"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="8"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="16"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="32"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="64"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="128"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="256"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="512"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="1024"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="2048"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="4096"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="8192"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="16384"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="32768"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="65536"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="131072"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="+Inf"} 2 +milvus_proxy_cache_update_latency_sum{cache_name="GetCollectionID",node_id="37"} 1 +milvus_proxy_cache_update_latency_count{cache_name="GetCollectionID",node_id="37"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="1"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="2"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="4"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="8"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="16"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="32"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="64"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="128"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="256"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="512"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="1024"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="2048"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="4096"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="8192"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="16384"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="32768"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="65536"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="131072"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="+Inf"} 1 +milvus_proxy_cache_update_latency_sum{cache_name="GetShards",node_id="37"} 1 +milvus_proxy_cache_update_latency_count{cache_name="GetShards",node_id="37"} 1 # HELP milvus_proxy_delete_vectors_count counter of vectors successfully deleted # TYPE milvus_proxy_delete_vectors_count counter -milvus_proxy_delete_vectors_count{db_name="default",node_id="2"} 2 +milvus_proxy_delete_vectors_count{collection_name="hello_milvus",db_name="default",node_id="37"} 2 # HELP milvus_proxy_msgstream_obj_num number of MsgStream objects per physical channel # TYPE milvus_proxy_msgstream_obj_num gauge -milvus_proxy_msgstream_obj_num{channel_name="by-dev-rootcoord-dml_0",node_id="2"} 0 +milvus_proxy_msgstream_obj_num{channel_name="by-dev-rootcoord-dml_10",node_id="37"} 0 # HELP milvus_proxy_mutation_send_latency latency that proxy send insert request to MsgStream # TYPE milvus_proxy_mutation_send_latency histogram -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="100"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="500"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="5000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="10000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="20000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="50000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="100000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="250000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="500000"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1e+06"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="3.6e+06"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="5e+06"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="1e+07"} 2 -milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="2",le="+Inf"} 2 -milvus_proxy_mutation_send_latency_sum{msg_type="insert",node_id="2"} 0 -milvus_proxy_mutation_send_latency_count{msg_type="insert",node_id="2"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="100"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="500"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="5000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="10000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="20000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="50000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="100000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="250000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="500000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="3.6e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="5e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1e+07"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="+Inf"} 2 +milvus_proxy_mutation_send_latency_sum{msg_type="insert",node_id="37"} 1 +milvus_proxy_mutation_send_latency_count{msg_type="insert",node_id="37"} 2 # HELP milvus_proxy_rate_limit_req_count count of operation executed # TYPE milvus_proxy_rate_limit_req_count counter -milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="2",status="success"} 27 -milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="2",status="total"} 27 -milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="2",status="success"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="2",status="total"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="2",status="success"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="2",status="total"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="2",status="success"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="2",status="total"} 1 -milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="2",status="success"} 2 -milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="2",status="total"} 2 -milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="2",status="success"} 5 -milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="2",status="total"} 5 -milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="2",status="success"} 2 -milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="2",status="total"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="37",status="success"} 23 +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="37",status="total"} 23 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="37",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="37",status="total"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="37",status="success"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="37",status="total"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="37",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="37",status="total"} 2 # HELP milvus_proxy_report_value report value about the request # TYPE milvus_proxy_report_value counter -milvus_proxy_report_value{db_name="default",msg_type="delete",node_id="2",username=""} 0 -milvus_proxy_report_value{db_name="default",msg_type="insert",node_id="2",username=""} 0 -milvus_proxy_report_value{db_name="default",msg_type="query",node_id="2",username=""} 0 -milvus_proxy_report_value{db_name="default",msg_type="search",node_id="2",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="delete",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="insert",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="query",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="search",node_id="37",username=""} 0 # HELP milvus_proxy_req_count count of operation executed # TYPE milvus_proxy_req_count counter -milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="2",status="success"} 1 -milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="2",status="total"} 1 -milvus_proxy_req_count{collection_name="hello_milvus",db_name="default",function_name="DropCollection",node_id="2",status="success"} 1 +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="37",status="success"} 1 +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="37",status="total"} 1 +milvus_proxy_req_count{collection_name="hello_milvus",db_name="default",function_name="DropCollection",node_id="37",status="success"} 1 # HELP milvus_proxy_req_in_queue_latency latency which request waits in the queue # TYPE milvus_proxy_req_in_queue_latency histogram -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="CreateCollection",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="CreateCollection",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="CreateIndex",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="CreateIndex",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="Delete",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="Delete",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="5"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="10"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="20"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="50"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="100"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="200"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="300"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="400"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="500"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="600"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="700"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="800"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="900"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="1000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="1500"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="3000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="10000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="60000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="2",le="+Inf"} 4 -milvus_proxy_req_in_queue_latency_sum{function_name="DescribeCollection",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="DescribeCollection",node_id="2"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="5"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="10"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="20"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="50"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="100"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="200"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="300"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="400"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="500"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="600"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="700"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="800"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="900"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="1000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="1500"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="3000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="10000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="60000"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="2",le="+Inf"} 4 -milvus_proxy_req_in_queue_latency_sum{function_name="DescribeIndex",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="DescribeIndex",node_id="2"} 4 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="DropCollection",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="DropCollection",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="Flush",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="Flush",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="GetCollectionStatistics",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="GetCollectionStatistics",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="5"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="10"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="20"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="50"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="100"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="200"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="300"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="400"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="500"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="600"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="700"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="800"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="900"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="1000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="1500"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="3000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="10000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="60000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="2",le="+Inf"} 2 -milvus_proxy_req_in_queue_latency_sum{function_name="Insert",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="Insert",node_id="2"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="5"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="10"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="20"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="50"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="100"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="200"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="300"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="400"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="600"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="700"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="800"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="900"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_in_queue_latency_sum{function_name="LoadCollection",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="LoadCollection",node_id="2"} 1 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="5"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="10"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="20"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="50"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="100"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="200"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="300"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="400"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="500"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="600"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="700"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="800"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="900"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="1000"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="1500"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="3000"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="10000"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="60000"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="2",le="+Inf"} 5 -milvus_proxy_req_in_queue_latency_sum{function_name="Retrieve",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="Retrieve",node_id="2"} 5 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="5"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="10"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="20"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="50"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="100"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="200"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="300"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="400"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="500"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="600"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="700"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="800"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="900"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="1000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="1500"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="3000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="10000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="60000"} 2 -milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="2",le="+Inf"} 2 -milvus_proxy_req_in_queue_latency_sum{function_name="Search",node_id="2"} 0 -milvus_proxy_req_in_queue_latency_count{function_name="Search",node_id="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateIndex",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateIndex",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Delete",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Delete",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="1"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="2"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="4"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="8"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="16"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="32"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="64"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="128"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="256"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="512"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="1024"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="2048"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="4096"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="8192"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="16384"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="32768"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="65536"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="131072"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="+Inf"} 3 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeCollection",node_id="37"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="1"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="2"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="4"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="8"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="16"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="32"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="64"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="128"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="256"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="512"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="1024"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="2048"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="4096"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="8192"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="16384"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="32768"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="65536"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="131072"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="+Inf"} 3 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeIndex",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeIndex",node_id="37"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="DropCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DropCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Flush",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Flush",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="1"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="4"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="8"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="16"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="32"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="64"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="128"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="256"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="512"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="1024"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="2048"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="4096"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="8192"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="16384"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="32768"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="65536"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="131072"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Insert",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Insert",node_id="37"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="LoadCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="LoadCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="1"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="2"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="4"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="8"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="16"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="32"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="64"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="128"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="256"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="512"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="1024"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="2048"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="4096"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="8192"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="16384"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="32768"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="65536"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="131072"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="+Inf"} 5 +milvus_proxy_req_in_queue_latency_sum{function_name="Retrieve",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Retrieve",node_id="37"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="1"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="4"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="8"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="16"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="32"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="64"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="128"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="256"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="512"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="1024"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="2048"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="4096"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="8192"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="16384"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="32768"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="65536"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="131072"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Search",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Search",node_id="37"} 2 # HELP milvus_proxy_req_latency latency of each request # TYPE milvus_proxy_req_latency histogram -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="5"} 0 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="CreateCollection",node_id="2"} 7 -milvus_proxy_req_latency_count{function_name="CreateCollection",node_id="2"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="5"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="CreateIndex",node_id="2"} 2 -milvus_proxy_req_latency_count{function_name="CreateIndex",node_id="2"} 1 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="5"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="10"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="20"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="50"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="100"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="200"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="300"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="400"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="500"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="600"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="700"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="800"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="900"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="1000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="1500"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="3000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="10000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="60000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="2",le="+Inf"} 4 -milvus_proxy_req_latency_sum{function_name="DescribeCollection",node_id="2"} 0 -milvus_proxy_req_latency_count{function_name="DescribeCollection",node_id="2"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="5"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="10"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="20"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="50"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="100"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="200"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="300"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="400"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="500"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="600"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="700"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="800"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="900"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="1000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="1500"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="3000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="10000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="60000"} 4 -milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="2",le="+Inf"} 4 -milvus_proxy_req_latency_sum{function_name="DescribeIndex",node_id="2"} 0 -milvus_proxy_req_latency_count{function_name="DescribeIndex",node_id="2"} 4 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="5"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="DropCollection",node_id="2"} 1 -milvus_proxy_req_latency_count{function_name="DropCollection",node_id="2"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="5"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="Flush",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="Flush",node_id="2"} 1 -milvus_proxy_req_latency_count{function_name="Flush",node_id="2"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="5"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="GetCollectionStatistics",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="GetCollectionStatistics",node_id="2"} 0 -milvus_proxy_req_latency_count{function_name="GetCollectionStatistics",node_id="2"} 1 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="5"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="10"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="20"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="50"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="100"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="200"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="300"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="400"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="500"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="600"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="700"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="800"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="900"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="1000"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="1500"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="3000"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="10000"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="60000"} 6 -milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="2",le="+Inf"} 6 -milvus_proxy_req_latency_sum{function_name="GetLoadingProgress",node_id="2"} 0 -milvus_proxy_req_latency_count{function_name="GetLoadingProgress",node_id="2"} 6 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="5"} 0 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="10"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="20"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="50"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="100"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="200"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="300"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="400"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="500"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="600"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="700"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="800"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="900"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="1000"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="1500"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="3000"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="10000"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="60000"} 1 -milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="2",le="+Inf"} 1 -milvus_proxy_req_latency_sum{function_name="LoadCollection",node_id="2"} 6 -milvus_proxy_req_latency_count{function_name="LoadCollection",node_id="2"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="4"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateCollection",node_id="37"} 4 +milvus_proxy_req_latency_count{function_name="CreateCollection",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateIndex",node_id="37"} 7 +milvus_proxy_req_latency_count{function_name="CreateIndex",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="1"} 2 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="2"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="4"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="8"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="16"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="32"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="64"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="128"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="256"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="512"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="1024"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="2048"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="4096"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="8192"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="16384"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="32768"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="65536"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="131072"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="+Inf"} 3 +milvus_proxy_req_latency_sum{function_name="DescribeCollection",node_id="37"} 2 +milvus_proxy_req_latency_count{function_name="DescribeCollection",node_id="37"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="2"} 2 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="4"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="8"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="16"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="32"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="64"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="128"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="256"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="512"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="1024"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="2048"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="4096"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="8192"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="16384"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="32768"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="65536"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="131072"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="+Inf"} 3 +milvus_proxy_req_latency_sum{function_name="DescribeIndex",node_id="37"} 8 +milvus_proxy_req_latency_count{function_name="DescribeIndex",node_id="37"} 3 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="DropCollection",node_id="37"} 6 +milvus_proxy_req_latency_count{function_name="DropCollection",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="8"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="16"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="32"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="64"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="128"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="256"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="512"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="Flush",node_id="37"} 619 +milvus_proxy_req_latency_count{function_name="Flush",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="1"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="2"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="4"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="8"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="16"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="32"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="64"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="128"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="256"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="512"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="1024"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="2048"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="4096"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="8192"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="16384"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="32768"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="65536"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="131072"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="+Inf"} 6 +milvus_proxy_req_latency_sum{function_name="GetLoadingProgress",node_id="37"} 2 +milvus_proxy_req_latency_count{function_name="GetLoadingProgress",node_id="37"} 6 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="LoadCollection",node_id="37"} 7 +milvus_proxy_req_latency_count{function_name="LoadCollection",node_id="37"} 1 +# HELP milvus_proxy_search_sparse_num_non_zeros the number of non-zeros in each sparse search task +# TYPE milvus_proxy_search_sparse_num_non_zeros histogram +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="1"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="2"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="4"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="8"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="16"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="32"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="64"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="128"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="256"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="512"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="1024"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="2048"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="4096"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="8192"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="16384"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="32768"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="65536"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="131072"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="+Inf"} 2 +milvus_proxy_search_sparse_num_non_zeros_sum{collection_name="hello_milvus",node_id="37"} 16 +milvus_proxy_search_sparse_num_non_zeros_count{collection_name="hello_milvus",node_id="37"} 2 # HELP milvus_proxy_send_bytes_count count of bytes sent back to sdk # TYPE milvus_proxy_send_bytes_count counter -milvus_proxy_send_bytes_count{node_id="2"} 70813 +milvus_proxy_send_bytes_count{node_id="37"} 70813 # HELP milvus_proxy_sq_decode_result_latency latency that proxy decodes the search result # TYPE milvus_proxy_sq_decode_result_latency histogram -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="5"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="10"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="20"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="50"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="100"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="200"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="300"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 -milvus_proxy_sq_decode_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_proxy_sq_decode_result_latency_sum{node_id="2",query_type="query"} 0 -milvus_proxy_sq_decode_result_latency_count{node_id="2",query_type="query"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="1"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="2"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_decode_result_latency_sum{node_id="37",query_type="query"} 0 +milvus_proxy_sq_decode_result_latency_count{node_id="37",query_type="query"} 5 # HELP milvus_proxy_sq_reduce_result_latency latency that proxy reduces search result # TYPE milvus_proxy_sq_reduce_result_latency histogram -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="5"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="10"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="20"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="50"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="100"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="200"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="300"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_proxy_sq_reduce_result_latency_sum{node_id="2",query_type="query"} 0 -milvus_proxy_sq_reduce_result_latency_count{node_id="2",query_type="query"} 5 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="5"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="10"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="20"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="50"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="100"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="200"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="300"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="400"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="500"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="600"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="700"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="800"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="900"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="1000"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="1500"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="3000"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="10000"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="60000"} 2 -milvus_proxy_sq_reduce_result_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 -milvus_proxy_sq_reduce_result_latency_sum{node_id="2",query_type="search"} 0 -milvus_proxy_sq_reduce_result_latency_count{node_id="2",query_type="search"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="1"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="2"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_reduce_result_latency_sum{node_id="37",query_type="query"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="37",query_type="query"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="1"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="2"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="4"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="8"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="16"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="32"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="64"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="128"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="256"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_reduce_result_latency_sum{node_id="37",query_type="search"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="37",query_type="search"} 2 # HELP milvus_proxy_sq_wait_result_latency latency that proxy waits for the result # TYPE milvus_proxy_sq_wait_result_latency histogram -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="5"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="10"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="20"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="50"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="100"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="200"} 4 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="300"} 4 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="400"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="600"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="700"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="800"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="900"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="1500"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="3000"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="60000"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_proxy_sq_wait_result_latency_sum{node_id="2",query_type="query"} 1132 -milvus_proxy_sq_wait_result_latency_count{node_id="2",query_type="query"} 5 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="5"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="10"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="20"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="50"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="100"} 0 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="200"} 1 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="300"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="400"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="500"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="600"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="700"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="800"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="900"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="1000"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="1500"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="3000"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="10000"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="60000"} 2 -milvus_proxy_sq_wait_result_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 -milvus_proxy_sq_wait_result_latency_sum{node_id="2",query_type="search"} 470 -milvus_proxy_sq_wait_result_latency_count{node_id="2",query_type="search"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="2"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="4"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="8"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="16"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="32"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="64"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="128"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="256"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_wait_result_latency_sum{node_id="37",query_type="query"} 1478 +milvus_proxy_sq_wait_result_latency_count{node_id="37",query_type="query"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="2"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="4"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="8"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="16"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="32"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="64"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="128"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="256"} 1 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_wait_result_latency_sum{node_id="37",query_type="search"} 610 +milvus_proxy_sq_wait_result_latency_count{node_id="37",query_type="search"} 2 # HELP milvus_proxy_sync_segment_request_length the length of SegmentIDRequests when assigning segments for insert # TYPE milvus_proxy_sync_segment_request_length histogram -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="5"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="10"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="20"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="50"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="100"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="200"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="300"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="400"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="500"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="600"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="700"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="800"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="900"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="1000"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="1500"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="3000"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="10000"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="60000"} 2 -milvus_proxy_sync_segment_request_length_bucket{node_id="2",le="+Inf"} 2 -milvus_proxy_sync_segment_request_length_sum{node_id="2"} 2 -milvus_proxy_sync_segment_request_length_count{node_id="2"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="1"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="2"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="4"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="8"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="16"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="32"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="64"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="128"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="256"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="512"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="1024"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="2048"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="4096"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="8192"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="16384"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="32768"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="65536"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="131072"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_sync_segment_request_length_sum{node_id="37"} 2 +milvus_proxy_sync_segment_request_length_count{node_id="37"} 2 # HELP milvus_proxy_tt_lag_ms now time minus tt per physical channel # TYPE milvus_proxy_tt_lag_ms gauge -milvus_proxy_tt_lag_ms{channel_name="default",node_id="2"} 141 +milvus_proxy_tt_lag_ms{channel_name="default",node_id="37"} 36 # HELP milvus_querycoord_collection_num number of collections # TYPE milvus_querycoord_collection_num gauge milvus_querycoord_collection_num 0 @@ -1563,7 +1692,7 @@ milvus_querycoord_load_latency_bucket{le="300000"} 2 milvus_querycoord_load_latency_bucket{le="600000"} 2 milvus_querycoord_load_latency_bucket{le="1.8e+06"} 2 milvus_querycoord_load_latency_bucket{le="+Inf"} 2 -milvus_querycoord_load_latency_sum 1805 +milvus_querycoord_load_latency_sum 1996 milvus_querycoord_load_latency_count 2 # HELP milvus_querycoord_load_req_count count of load request # TYPE milvus_querycoord_load_req_count counter @@ -1577,23 +1706,32 @@ milvus_querycoord_partition_num 0 milvus_querycoord_querynode_num 1 # HELP milvus_querycoord_release_latency latency of release request # TYPE milvus_querycoord_release_latency histogram -milvus_querycoord_release_latency_bucket{le="0"} 0 -milvus_querycoord_release_latency_bucket{le="5"} 0 -milvus_querycoord_release_latency_bucket{le="10"} 0 -milvus_querycoord_release_latency_bucket{le="20"} 0 -milvus_querycoord_release_latency_bucket{le="40"} 0 -milvus_querycoord_release_latency_bucket{le="100"} 0 -milvus_querycoord_release_latency_bucket{le="200"} 0 -milvus_querycoord_release_latency_bucket{le="400"} 0 -milvus_querycoord_release_latency_bucket{le="1000"} 1 -milvus_querycoord_release_latency_bucket{le="10000"} 1 -milvus_querycoord_release_latency_bucket{le="+Inf"} 1 -milvus_querycoord_release_latency_sum 605 -milvus_querycoord_release_latency_count 1 +milvus_querycoord_release_latency_bucket{le="0"} 10 +milvus_querycoord_release_latency_bucket{le="5"} 10 +milvus_querycoord_release_latency_bucket{le="10"} 10 +milvus_querycoord_release_latency_bucket{le="20"} 10 +milvus_querycoord_release_latency_bucket{le="40"} 10 +milvus_querycoord_release_latency_bucket{le="100"} 10 +milvus_querycoord_release_latency_bucket{le="200"} 10 +milvus_querycoord_release_latency_bucket{le="400"} 10 +milvus_querycoord_release_latency_bucket{le="1000"} 11 +milvus_querycoord_release_latency_bucket{le="10000"} 11 +milvus_querycoord_release_latency_bucket{le="+Inf"} 11 +milvus_querycoord_release_latency_sum 606 +milvus_querycoord_release_latency_count 11 # HELP milvus_querycoord_release_req_count count of release request # TYPE milvus_querycoord_release_req_count counter milvus_querycoord_release_req_count{status="success"} 1 milvus_querycoord_release_req_count{status="total"} 1 +# HELP milvus_querycoord_replica_ro_node_total total read only node number of replica +# TYPE milvus_querycoord_replica_ro_node_total gauge +milvus_querycoord_replica_ro_node_total 0 +# HELP milvus_querycoord_resource_group_info all resource group detail info in query coord +# TYPE milvus_querycoord_resource_group_info gauge +milvus_querycoord_resource_group_info{node_id="37",rg="__default_resource_group"} 1 +# HELP milvus_querycoord_resource_group_replica_total total replica number of resource group +# TYPE milvus_querycoord_resource_group_replica_total gauge +milvus_querycoord_resource_group_replica_total{rg="__default_resource_group"} 0 # HELP milvus_querycoord_task_num the number of tasks in QueryCoord's scheduler # TYPE milvus_querycoord_task_num gauge milvus_querycoord_task_num{querycoord_task_type="channel_grow"} 0 @@ -1604,867 +1742,916 @@ milvus_querycoord_task_num{querycoord_task_type="segment_move"} 0 milvus_querycoord_task_num{querycoord_task_type="segment_reduce"} 0 # HELP milvus_querynode_apply_bf_latency apply bf cost in ms # TYPE milvus_querynode_apply_bf_latency histogram -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="5"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="10"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="20"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="50"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="100"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="200"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="300"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="400"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="500"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="600"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="700"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="800"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="900"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="1000"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="1500"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="3000"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="10000"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="60000"} 1 -milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="2",le="+Inf"} 1 -milvus_querynode_apply_bf_latency_sum{function_name="ProcessDelete",node_id="2"} 0 -milvus_querynode_apply_bf_latency_count{function_name="ProcessDelete",node_id="2"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="1"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="2"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="4"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="8"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="16"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="32"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="64"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="128"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="256"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="512"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="1024"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="2048"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="4096"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="8192"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="16384"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="32768"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="65536"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="131072"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="+Inf"} 1 +milvus_querynode_apply_bf_latency_sum{function_name="ProcessDelete",node_id="37"} 0 +milvus_querynode_apply_bf_latency_count{function_name="ProcessDelete",node_id="37"} 1 +# HELP milvus_querynode_cgo_latency latency of each cgo call +# TYPE milvus_querynode_cgo_latency histogram +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="1"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="2"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="4"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="8"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="16"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="32"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="64"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="128"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="256"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="512"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="1024"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="2048"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="4096"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="8192"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="16384"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="32768"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="65536"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="131072"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="+Inf"} 1 +milvus_querynode_cgo_latency_sum{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_count{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="1"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="2"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="4"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="8"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="16"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="32"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="64"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="128"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="256"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="512"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="1024"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="2048"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="4096"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="8192"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="16384"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="32768"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="65536"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="131072"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="+Inf"} 1 +milvus_querynode_cgo_latency_sum{cgo_name="Delete",cgo_type="Sync",node_id="37"} 0 +milvus_querynode_cgo_latency_count{cgo_name="Delete",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="1"} 3 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="2"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="4"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="8"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="16"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="32"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="64"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="128"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="256"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="512"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="1024"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="2048"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="4096"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="8192"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="16384"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="32768"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="65536"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="131072"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="+Inf"} 4 +milvus_querynode_cgo_latency_sum{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37"} 5 +milvus_querynode_cgo_latency_count{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37"} 4 # HELP milvus_querynode_collection_num number of collections loaded # TYPE milvus_querynode_collection_num gauge -milvus_querynode_collection_num{node_id="2"} 0 +milvus_querynode_collection_num{node_id="37"} 0 # HELP milvus_querynode_consume_bytes_counter # TYPE milvus_querynode_consume_bytes_counter counter -milvus_querynode_consume_bytes_counter{msg_type="insert",node_id="2"} 231 +milvus_querynode_consume_bytes_counter{msg_type="insert",node_id="37"} 241 # HELP milvus_querynode_disk_cache_evict_bytes number of bytes evicted from disk cache # TYPE milvus_querynode_disk_cache_evict_bytes counter -milvus_querynode_disk_cache_evict_bytes{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_evict_bytes{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_cache_evict_duration total time cost of evicting segments from disk cache # TYPE milvus_querynode_disk_cache_evict_duration counter -milvus_querynode_disk_cache_evict_duration{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_evict_duration{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_cache_evict_global_duration global duration of evicting segments from disk cache # TYPE milvus_querynode_disk_cache_evict_global_duration histogram -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="100"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="500"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="5000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="10000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="20000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="50000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="100000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="250000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="500000"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1e+06"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="3.6e+06"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="5e+06"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="1e+07"} 0 -milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="2",le="+Inf"} 0 -milvus_querynode_disk_cache_evict_global_duration_sum{node_id="2"} 0 -milvus_querynode_disk_cache_evict_global_duration_count{node_id="2"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="100"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="500"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="5000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="10000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="20000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="50000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="100000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="250000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="500000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="3.6e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="5e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1e+07"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="+Inf"} 0 +milvus_querynode_disk_cache_evict_global_duration_sum{node_id="37"} 0 +milvus_querynode_disk_cache_evict_global_duration_count{node_id="37"} 0 # HELP milvus_querynode_disk_cache_evict_total number of segments evicted from disk cache # TYPE milvus_querynode_disk_cache_evict_total counter -milvus_querynode_disk_cache_evict_total{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_evict_total{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_cache_load_bytes number of bytes loaded from disk cache # TYPE milvus_querynode_disk_cache_load_bytes counter -milvus_querynode_disk_cache_load_bytes{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_load_bytes{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_cache_load_duration total time cost of loading segments from disk cache # TYPE milvus_querynode_disk_cache_load_duration counter -milvus_querynode_disk_cache_load_duration{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_load_duration{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_cache_load_global_duration global duration of loading segments from disk cache # TYPE milvus_querynode_disk_cache_load_global_duration histogram -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="100"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="500"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="5000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="10000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="20000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="50000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="100000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="250000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="500000"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1e+06"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="3.6e+06"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="5e+06"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="1e+07"} 0 -milvus_querynode_disk_cache_load_global_duration_bucket{node_id="2",le="+Inf"} 0 -milvus_querynode_disk_cache_load_global_duration_sum{node_id="2"} 0 -milvus_querynode_disk_cache_load_global_duration_count{node_id="2"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="100"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="500"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="5000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="10000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="20000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="50000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="100000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="250000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="500000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="3.6e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="5e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1e+07"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="+Inf"} 0 +milvus_querynode_disk_cache_load_global_duration_sum{node_id="37"} 0 +milvus_querynode_disk_cache_load_global_duration_count{node_id="37"} 0 # HELP milvus_querynode_disk_cache_load_total number of segments loaded from disk cache # TYPE milvus_querynode_disk_cache_load_total counter -milvus_querynode_disk_cache_load_total{db_name="default",node_id="2",rg="__default_resource_group"} 0 +milvus_querynode_disk_cache_load_total{db_name="default",node_id="37",rg="__default_resource_group"} 0 # HELP milvus_querynode_disk_used_size disk used size(MB) # TYPE milvus_querynode_disk_used_size gauge -milvus_querynode_disk_used_size{node_id="2"} 0.2461862564086914 +milvus_querynode_disk_used_size{node_id="37"} 4.285002708435059 # HELP milvus_querynode_dml_vchannel_num number of dmlChannels watched # TYPE milvus_querynode_dml_vchannel_num gauge -milvus_querynode_dml_vchannel_num{node_id="2"} 0 +milvus_querynode_dml_vchannel_num{node_id="37"} 0 # HELP milvus_querynode_execute_bytes_counter # TYPE milvus_querynode_execute_bytes_counter counter -milvus_querynode_execute_bytes_counter{msg_type="query",node_id="2"} 813 -milvus_querynode_execute_bytes_counter{msg_type="search",node_id="2"} 553 +milvus_querynode_execute_bytes_counter{msg_type="query",node_id="37"} 818 +milvus_querynode_execute_bytes_counter{msg_type="search",node_id="37"} 599 # HELP milvus_querynode_flowgraph_num number of flowgraphs # TYPE milvus_querynode_flowgraph_num gauge -milvus_querynode_flowgraph_num{node_id="2"} 0 +milvus_querynode_flowgraph_num{node_id="37"} 0 # HELP milvus_querynode_forward_delete_latency forward delete cost in ms # TYPE milvus_querynode_forward_delete_latency histogram -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="5"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="10"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="20"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="50"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="100"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="200"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="300"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="400"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="500"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="600"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="700"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="800"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="900"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="1000"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="1500"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="3000"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="10000"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="60000"} 1 -milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="2",le="+Inf"} 1 -milvus_querynode_forward_delete_latency_sum{function_name="ProcessDelete",node_id="2"} 0 -milvus_querynode_forward_delete_latency_count{function_name="ProcessDelete",node_id="2"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="1"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="2"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="4"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="8"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="16"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="32"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="64"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="128"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="256"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="512"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="1024"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="2048"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="4096"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="8192"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="16384"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="32768"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="65536"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="131072"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="+Inf"} 1 +milvus_querynode_forward_delete_latency_sum{function_name="ProcessDelete",node_id="37"} 1 +milvus_querynode_forward_delete_latency_count{function_name="ProcessDelete",node_id="37"} 1 # HELP milvus_querynode_load_index_latency latency of load per segment's index, in milliseconds # TYPE milvus_querynode_load_index_latency histogram -milvus_querynode_load_index_latency_bucket{node_id="2",le="1"} 0 -milvus_querynode_load_index_latency_bucket{node_id="2",le="100"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="500"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="1000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="5000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="10000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="20000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="50000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="100000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="250000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="500000"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="1e+06"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="3.6e+06"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="5e+06"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="1e+07"} 1 -milvus_querynode_load_index_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_querynode_load_index_latency_sum{node_id="2"} 3 -milvus_querynode_load_index_latency_count{node_id="2"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1"} 0 +milvus_querynode_load_index_latency_bucket{node_id="37",le="100"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="500"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="20000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="50000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="100000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="250000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="500000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="3.6e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="5e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1e+07"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_load_index_latency_sum{node_id="37"} 2 +milvus_querynode_load_index_latency_count{node_id="37"} 1 # HELP milvus_querynode_load_segment_concurrency number of concurrent loading segments in QueryNode # TYPE milvus_querynode_load_segment_concurrency gauge -milvus_querynode_load_segment_concurrency{load_type="LoadSegment",node_id="2"} 0 +milvus_querynode_load_segment_concurrency{load_type="LoadSegment",node_id="37"} 0 # HELP milvus_querynode_load_segment_latency latency of load per segment # TYPE milvus_querynode_load_segment_latency histogram -milvus_querynode_load_segment_latency_bucket{node_id="2",le="1"} 0 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="100"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="500"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="1000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="5000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="10000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="20000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="50000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="100000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="250000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="500000"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="1e+06"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="3.6e+06"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="5e+06"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="1e+07"} 1 -milvus_querynode_load_segment_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_querynode_load_segment_latency_sum{node_id="2"} 22 -milvus_querynode_load_segment_latency_count{node_id="2"} 1 -# HELP milvus_querynode_partition_num number of partitions loaded -# TYPE milvus_querynode_partition_num gauge -milvus_querynode_partition_num{node_id="2"} 0 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1"} 0 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="100"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="500"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="5000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="10000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="20000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="50000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="100000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="250000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="500000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="3.6e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="5e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1e+07"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_load_segment_latency_sum{node_id="37"} 9 +milvus_querynode_load_segment_latency_count{node_id="37"} 1 # HELP milvus_querynode_process_insert_or_delete_latency process insert or delete cost in ms # TYPE milvus_querynode_process_insert_or_delete_latency histogram -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="5"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="10"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="20"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="50"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="100"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="200"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="300"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="400"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="500"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="600"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="700"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="800"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="900"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="1000"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="1500"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="3000"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="10000"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="60000"} 1 -milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="2",le="+Inf"} 1 -milvus_querynode_process_insert_or_delete_latency_sum{msg_type="delete",node_id="2"} 1 -milvus_querynode_process_insert_or_delete_latency_count{msg_type="delete",node_id="2"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="1"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="2"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="4"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="8"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="16"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="32"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="64"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="128"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="256"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="512"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="1024"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="2048"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="4096"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="8192"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="16384"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="32768"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="65536"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="131072"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="+Inf"} 1 +milvus_querynode_process_insert_or_delete_latency_sum{msg_type="delete",node_id="37"} 1 +milvus_querynode_process_insert_or_delete_latency_count{msg_type="delete",node_id="37"} 1 # HELP milvus_querynode_read_task_concurrency number of concurrent executing read tasks in QueryNode # TYPE milvus_querynode_read_task_concurrency gauge -milvus_querynode_read_task_concurrency{node_id="2"} 0 +milvus_querynode_read_task_concurrency{node_id="37"} 0 # HELP milvus_querynode_read_task_ready_len number of ready read tasks in readyQueue # TYPE milvus_querynode_read_task_ready_len gauge -milvus_querynode_read_task_ready_len{node_id="2"} 0 +milvus_querynode_read_task_ready_len{node_id="37"} 0 # HELP milvus_querynode_read_task_unsolved_len number of unsolved read tasks in unsolvedQueue # TYPE milvus_querynode_read_task_unsolved_len gauge -milvus_querynode_read_task_unsolved_len{node_id="2"} 0 +milvus_querynode_read_task_unsolved_len{node_id="37"} 0 # HELP milvus_querynode_search_group_nq the number of queries of each grouped search task # TYPE milvus_querynode_search_group_nq histogram -milvus_querynode_search_group_nq_bucket{node_id="2",le="5"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="10"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="20"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="50"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="100"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="200"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="300"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="400"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="500"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="600"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="700"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="800"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="900"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="1000"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="1500"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="3000"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="10000"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="60000"} 2 -milvus_querynode_search_group_nq_bucket{node_id="2",le="+Inf"} 2 -milvus_querynode_search_group_nq_sum{node_id="2"} 4 -milvus_querynode_search_group_nq_count{node_id="2"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_group_nq_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_nq_sum{node_id="37"} 4 +milvus_querynode_search_group_nq_count{node_id="37"} 2 # HELP milvus_querynode_search_group_size the number of tasks of each grouped search task # TYPE milvus_querynode_search_group_size histogram -milvus_querynode_search_group_size_bucket{node_id="2",le="5"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="10"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="20"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="50"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="100"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="200"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="300"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="400"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="500"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="600"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="700"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="800"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="900"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="1000"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="1500"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="3000"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="10000"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="60000"} 2 -milvus_querynode_search_group_size_bucket{node_id="2",le="+Inf"} 2 -milvus_querynode_search_group_size_sum{node_id="2"} 2 -milvus_querynode_search_group_size_count{node_id="2"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="1"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_size_sum{node_id="37"} 2 +milvus_querynode_search_group_size_count{node_id="37"} 2 # HELP milvus_querynode_search_group_topk the topK of each grouped search task # TYPE milvus_querynode_search_group_topk histogram -milvus_querynode_search_group_topk_bucket{node_id="2",le="5"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="10"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="20"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="50"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="100"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="200"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="300"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="400"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="500"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="600"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="700"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="800"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="900"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="1000"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="1500"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="3000"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="10000"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="60000"} 2 -milvus_querynode_search_group_topk_bucket{node_id="2",le="+Inf"} 2 -milvus_querynode_search_group_topk_sum{node_id="2"} 6 -milvus_querynode_search_group_topk_count{node_id="2"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_group_topk_bucket{node_id="37",le="2"} 0 +milvus_querynode_search_group_topk_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_topk_sum{node_id="37"} 6 +milvus_querynode_search_group_topk_count{node_id="37"} 2 # HELP milvus_querynode_search_nq the number of queries of each search task # TYPE milvus_querynode_search_nq histogram -milvus_querynode_search_nq_bucket{node_id="2",le="5"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="10"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="20"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="50"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="100"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="200"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="300"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="400"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="500"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="600"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="700"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="800"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="900"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="1000"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="1500"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="3000"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="10000"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="60000"} 2 -milvus_querynode_search_nq_bucket{node_id="2",le="+Inf"} 2 -milvus_querynode_search_nq_sum{node_id="2"} 4 -milvus_querynode_search_nq_count{node_id="2"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_nq_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_nq_sum{node_id="37"} 4 +milvus_querynode_search_nq_count{node_id="37"} 2 # HELP milvus_querynode_search_topk the top of each search task # TYPE milvus_querynode_search_topk histogram -milvus_querynode_search_topk_bucket{node_id="2",le="5"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="10"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="20"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="50"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="100"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="200"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="300"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="400"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="500"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="600"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="700"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="800"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="900"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="1000"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="1500"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="3000"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="10000"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="60000"} 2 -milvus_querynode_search_topk_bucket{node_id="2",le="+Inf"} 2 -milvus_querynode_search_topk_sum{node_id="2"} 6 -milvus_querynode_search_topk_count{node_id="2"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_topk_bucket{node_id="37",le="2"} 0 +milvus_querynode_search_topk_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_topk_sum{node_id="37"} 6 +milvus_querynode_search_topk_count{node_id="37"} 2 # HELP milvus_querynode_segment_access_duration total time cost of accessing segments # TYPE milvus_querynode_segment_access_duration counter -milvus_querynode_segment_access_duration{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 3.8771669999999996 -milvus_querynode_segment_access_duration{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2.114125 +milvus_querynode_segment_access_duration{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 11.942583 +milvus_querynode_segment_access_duration{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 4.55725 # HELP milvus_querynode_segment_access_global_duration global time cost of accessing segments # TYPE milvus_querynode_segment_access_global_duration histogram -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1"} 3 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="100"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="5000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="20000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="50000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="100000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="250000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="500000"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1e+06"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="3.6e+06"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="5e+06"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="1e+07"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_querynode_segment_access_global_duration_sum{node_id="2",query_type="query"} 3.8771669999999996 -milvus_querynode_segment_access_global_duration_count{node_id="2",query_type="query"} 5 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1"} 1 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="100"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="500"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="5000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="10000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="20000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="50000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="100000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="250000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="500000"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1e+06"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="3.6e+06"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="5e+06"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="1e+07"} 2 -milvus_querynode_segment_access_global_duration_bucket{node_id="2",query_type="search",le="+Inf"} 2 -milvus_querynode_segment_access_global_duration_sum{node_id="2",query_type="search"} 2.114125 -milvus_querynode_segment_access_global_duration_count{node_id="2",query_type="search"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="100"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="500"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="5000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="10000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="20000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="50000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="100000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="250000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="500000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="3.6e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="5e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1e+07"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_segment_access_global_duration_sum{node_id="37",query_type="query"} 11.942583 +milvus_querynode_segment_access_global_duration_count{node_id="37",query_type="query"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="100"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="500"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="5000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="10000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="20000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="50000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="100000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="250000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="500000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="3.6e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="5e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1e+07"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_segment_access_global_duration_sum{node_id="37",query_type="search"} 4.55725 +milvus_querynode_segment_access_global_duration_count{node_id="37",query_type="search"} 2 # HELP milvus_querynode_segment_access_total number of segments accessed # TYPE milvus_querynode_segment_access_total counter -milvus_querynode_segment_access_total{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 5 -milvus_querynode_segment_access_total{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2 +milvus_querynode_segment_access_total{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_segment_access_total{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 2 # HELP milvus_querynode_segment_access_wait_cache_duration total time cost of waiting for loading access # TYPE milvus_querynode_segment_access_wait_cache_duration counter -milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0 -milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0 # HELP milvus_querynode_segment_access_wait_cache_global_duration global time cost of waiting for loading access # TYPE milvus_querynode_segment_access_wait_cache_global_duration histogram -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="100"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="500"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="5000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="10000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="20000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="50000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="100000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="250000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="500000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="3.6e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="5e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="1e+07"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="query",le="+Inf"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="2",query_type="query"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="2",query_type="query"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="100"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="500"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="5000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="10000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="20000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="50000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="100000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="250000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="500000"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="3.6e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="5e+06"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="1e+07"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="2",query_type="search",le="+Inf"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="2",query_type="search"} 0 -milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="2",query_type="search"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="37",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="37",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="37",query_type="search"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="37",query_type="search"} 0 # HELP milvus_querynode_segment_access_wait_cache_total number of segments waiting for loading access # TYPE milvus_querynode_segment_access_wait_cache_total counter -milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0 -milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0 # HELP milvus_querynode_segment_latency_per_vector one vector's search latency per segment # TYPE milvus_querynode_segment_latency_per_vector histogram -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="5"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="20"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="50"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="100"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="200"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="300"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="400"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="500"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="600"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="700"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="800"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="900"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1000"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1500"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="3000"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10000"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="60000"} 2 -milvus_querynode_segment_latency_per_vector_bucket{node_id="2",query_type="search",segment_state="Sealed",le="+Inf"} 2 -milvus_querynode_segment_latency_per_vector_sum{node_id="2",query_type="search",segment_state="Sealed"} 0.5 -milvus_querynode_segment_latency_per_vector_count{node_id="2",query_type="search",segment_state="Sealed"} 2 -# HELP milvus_querynode_segment_num number of segments loaded, clustered by its collection, partition, state and # of indexed fields -# TYPE milvus_querynode_segment_num gauge -milvus_querynode_segment_num{collection_id="454172268311087164",indexed_field_count="1",node_id="2",partition_id="454172268311087165",segment_level="L1",segment_state="Sealed"} 0 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="64"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="128"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="256"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="512"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1024"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2048"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4096"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8192"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16384"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32768"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="65536"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="131072"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_segment_latency_per_vector_sum{node_id="37",query_type="search",segment_state="Sealed"} 1.5 +milvus_querynode_segment_latency_per_vector_count{node_id="37",query_type="search",segment_state="Sealed"} 2 # HELP milvus_querynode_sq_core_latency latency of search or query latency in segcore # TYPE milvus_querynode_sq_core_latency histogram -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="5"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="10"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="20"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="50"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="100"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="200"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="300"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="400"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="600"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="700"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="800"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="900"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="1500"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="3000"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="60000"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_querynode_sq_core_latency_sum{node_id="2",query_type="query"} 2 -milvus_querynode_sq_core_latency_count{node_id="2",query_type="query"} 5 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="5"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="10"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="20"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="50"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="100"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="200"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="300"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="400"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="500"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="600"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="700"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="800"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="900"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="1000"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="1500"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="3000"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="10000"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="60000"} 2 -milvus_querynode_sq_core_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 -milvus_querynode_sq_core_latency_sum{node_id="2",query_type="search"} 1 -milvus_querynode_sq_core_latency_count{node_id="2",query_type="search"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="1"} 4 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="2"} 4 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_core_latency_sum{node_id="37",query_type="query"} 7 +milvus_querynode_sq_core_latency_count{node_id="37",query_type="query"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="1"} 1 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="2"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="4"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="8"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="16"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="32"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="64"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="128"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="256"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_core_latency_sum{node_id="37",query_type="search"} 3 +milvus_querynode_sq_core_latency_count{node_id="37",query_type="search"} 2 # HELP milvus_querynode_sq_queue_latency latency of search or query in queue # TYPE milvus_querynode_sq_queue_latency histogram -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="5"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="10"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="20"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="50"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="100"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="200"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="300"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="400"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="500"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="600"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="700"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="800"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="900"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="1000"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="1500"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="3000"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="10000"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="60000"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="query",rg="__default_resource_group",le="+Inf"} 5 -milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 0.040208 -milvus_querynode_sq_queue_latency_count{db_name="default",node_id="2",query_type="query",rg="__default_resource_group"} 5 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="5"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="10"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="20"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="50"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="100"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="200"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="300"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="400"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="500"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="600"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="700"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="800"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="900"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="1000"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="1500"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="3000"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="10000"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="60000"} 2 -milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="2",query_type="search",rg="__default_resource_group",le="+Inf"} 2 -milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 0.040541 -milvus_querynode_sq_queue_latency_count{db_name="default",node_id="2",query_type="search",rg="__default_resource_group"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="1"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="2"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="4"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="8"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="16"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="32"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="64"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="128"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="256"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="512"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="1024"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="2048"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="4096"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="8192"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="16384"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="32768"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="65536"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="131072"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="+Inf"} 5 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0.39716599999999996 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="1"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="2"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="4"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="8"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="16"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="32"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="64"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="128"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="256"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="512"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="1024"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="2048"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="4096"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="8192"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="16384"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="32768"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="65536"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="131072"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="+Inf"} 2 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0.12625 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 2 # HELP milvus_querynode_sq_queue_user_latency latency per user of search or query in queue # TYPE milvus_querynode_sq_queue_user_latency histogram -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="5"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="10"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="20"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="50"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="100"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="200"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="300"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="400"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="500"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="600"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="700"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="800"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="900"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="1000"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="1500"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="3000"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="10000"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="60000"} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="query",username="",le="+Inf"} 5 -milvus_querynode_sq_queue_user_latency_sum{node_id="2",query_type="query",username=""} 0.040208 -milvus_querynode_sq_queue_user_latency_count{node_id="2",query_type="query",username=""} 5 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="5"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="10"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="20"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="50"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="100"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="200"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="300"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="400"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="500"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="600"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="700"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="800"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="900"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="1000"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="1500"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="3000"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="10000"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="60000"} 2 -milvus_querynode_sq_queue_user_latency_bucket{node_id="2",query_type="search",username="",le="+Inf"} 2 -milvus_querynode_sq_queue_user_latency_sum{node_id="2",query_type="search",username=""} 0.040541 -milvus_querynode_sq_queue_user_latency_count{node_id="2",query_type="search",username=""} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="1"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="2"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="4"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="8"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="16"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="32"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="64"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="128"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="256"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="512"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="1024"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="2048"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="4096"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="8192"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="16384"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="32768"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="65536"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="131072"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="+Inf"} 5 +milvus_querynode_sq_queue_user_latency_sum{node_id="37",query_type="query",username=""} 0.39716599999999996 +milvus_querynode_sq_queue_user_latency_count{node_id="37",query_type="query",username=""} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="1"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="2"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="4"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="8"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="16"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="32"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="64"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="128"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="256"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="512"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="1024"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="2048"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="4096"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="8192"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="16384"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="32768"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="65536"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="131072"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="+Inf"} 2 +milvus_querynode_sq_queue_user_latency_sum{node_id="37",query_type="search",username=""} 0.12625 +milvus_querynode_sq_queue_user_latency_count{node_id="37",query_type="search",username=""} 2 # HELP milvus_querynode_sq_reduce_latency latency of reduce search or query result # TYPE milvus_querynode_sq_reduce_latency histogram -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="5"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="10"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="20"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="50"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="100"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="200"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="300"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="400"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="500"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="600"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="700"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="800"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="900"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1500"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="3000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="10000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="60000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 5 -milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 0 -milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="5"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="10"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="20"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="50"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="100"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="200"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="300"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="400"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="500"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="600"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="700"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="800"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="900"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1500"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="3000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="10000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="60000"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 5 -milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 0 -milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 5 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="5"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="10"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="20"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="50"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="100"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="200"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="300"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="400"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="500"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="600"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="700"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="800"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="900"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1500"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="3000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="10000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="60000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 2 -milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 0 -milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="5"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="10"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="20"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="50"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="100"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="200"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="300"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="400"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="500"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="600"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="700"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="800"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="900"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1500"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="3000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="10000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="60000"} 2 -milvus_querynode_sq_reduce_latency_bucket{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 2 -milvus_querynode_sq_reduce_latency_sum{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 0 -milvus_querynode_sq_reduce_latency_count{node_id="2",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 2 -# HELP milvus_querynode_sq_req_count count of search / query request -# TYPE milvus_querynode_sq_req_count counter -milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="FromLeader",status="success"} 5 -milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="FromLeader",status="total"} 5 -milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="OnLeader",status="success"} 5 -milvus_querynode_sq_req_count{node_id="2",query_type="query",scope="OnLeader",status="total"} 5 -milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="FromLeader",status="success"} 2 -milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="FromLeader",status="total"} 2 -milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="OnLeader",status="success"} 2 -milvus_querynode_sq_req_count{node_id="2",query_type="search",scope="OnLeader",status="total"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="2"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="4"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="8"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="16"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="32"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="64"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="128"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="256"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="512"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1024"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="2048"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="4096"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="8192"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="16384"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="32768"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="65536"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="131072"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 1 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="2"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="4"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="8"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="16"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="32"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="64"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="128"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="256"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="512"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1024"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="2048"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="4096"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="8192"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="16384"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="32768"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="65536"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="131072"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="2"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="4"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="8"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="16"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="32"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="64"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="128"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="256"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="512"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1024"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="2048"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="4096"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="8192"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="16384"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="32768"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="65536"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="131072"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="2"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="4"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="8"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="16"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="32"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="64"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="128"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="256"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="512"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1024"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="2048"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="4096"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="8192"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="16384"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="32768"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="65536"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="131072"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 2 # HELP milvus_querynode_sq_req_latency latency of Search or query requests # TYPE milvus_querynode_sq_req_latency histogram -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="5"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="10"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="20"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="50"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="100"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="200"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="300"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="400"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="500"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="600"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="700"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="800"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="900"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="1000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="1500"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="3000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="10000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="60000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="FromLeader",le="+Inf"} 5 -milvus_querynode_sq_req_latency_sum{node_id="2",query_type="query",scope="FromLeader"} 3 -milvus_querynode_sq_req_latency_count{node_id="2",query_type="query",scope="FromLeader"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="5"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="10"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="20"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="50"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="100"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="200"} 4 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="300"} 4 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="400"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="500"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="600"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="700"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="800"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="900"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="1000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="1500"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="3000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="10000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="60000"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="query",scope="OnLeader",le="+Inf"} 5 -milvus_querynode_sq_req_latency_sum{node_id="2",query_type="query",scope="OnLeader"} 1127 -milvus_querynode_sq_req_latency_count{node_id="2",query_type="query",scope="OnLeader"} 5 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="5"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="10"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="20"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="50"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="100"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="200"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="300"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="400"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="500"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="600"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="700"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="800"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="900"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="1000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="1500"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="3000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="10000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="60000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="FromLeader",le="+Inf"} 2 -milvus_querynode_sq_req_latency_sum{node_id="2",query_type="search",scope="FromLeader"} 2 -milvus_querynode_sq_req_latency_count{node_id="2",query_type="search",scope="FromLeader"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="5"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="10"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="20"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="50"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="100"} 0 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="200"} 1 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="300"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="400"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="500"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="600"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="700"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="800"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="900"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="1000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="1500"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="3000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="10000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="60000"} 2 -milvus_querynode_sq_req_latency_bucket{node_id="2",query_type="search",scope="OnLeader",le="+Inf"} 2 -milvus_querynode_sq_req_latency_sum{node_id="2",query_type="search",scope="OnLeader"} 470 -milvus_querynode_sq_req_latency_count{node_id="2",query_type="search",scope="OnLeader"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="1"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="2"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="4"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="8"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="16"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="32"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="64"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="128"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="256"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="512"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="1024"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="2048"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="4096"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="8192"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="16384"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="32768"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="65536"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="131072"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="query",scope="FromLeader"} 14 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="query",scope="FromLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="2"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="4"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="8"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="16"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="32"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="64"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="128"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="256"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="512"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="1024"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="2048"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="4096"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="8192"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="16384"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="32768"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="65536"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="131072"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="query",scope="OnLeader"} 1469 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="query",scope="OnLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="2"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="4"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="8"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="16"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="32"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="64"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="128"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="256"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="512"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="1024"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="2048"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="4096"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="8192"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="16384"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="32768"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="65536"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="131072"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="search",scope="FromLeader"} 6 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="search",scope="FromLeader"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="2"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="4"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="8"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="16"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="32"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="64"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="128"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="256"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="512"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="1024"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="2048"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="4096"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="8192"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="16384"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="32768"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="65536"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="131072"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="search",scope="OnLeader"} 405 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="search",scope="OnLeader"} 2 # HELP milvus_querynode_sq_segment_latency latency of search or query per segment # TYPE milvus_querynode_sq_segment_latency histogram -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="5"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="10"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="20"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="50"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="100"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="200"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="300"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="400"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="500"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="600"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="700"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="800"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="900"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="1000"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="1500"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="3000"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="10000"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="60000"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="query",segment_state="Sealed",le="+Inf"} 5 -milvus_querynode_sq_segment_latency_sum{node_id="2",query_type="query",segment_state="Sealed"} 2 -milvus_querynode_sq_segment_latency_count{node_id="2",query_type="query",segment_state="Sealed"} 5 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="5"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="20"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="50"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="100"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="200"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="300"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="400"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="500"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="600"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="700"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="800"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="900"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1000"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="1500"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="3000"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="10000"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="60000"} 2 -milvus_querynode_sq_segment_latency_bucket{node_id="2",query_type="search",segment_state="Sealed",le="+Inf"} 2 -milvus_querynode_sq_segment_latency_sum{node_id="2",query_type="search",segment_state="Sealed"} 1 -milvus_querynode_sq_segment_latency_count{node_id="2",query_type="search",segment_state="Sealed"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="1"} 3 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="2"} 4 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="4"} 4 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="8"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="16"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="32"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="64"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="128"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="256"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="512"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="1024"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="2048"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="4096"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="8192"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="16384"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="32768"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="65536"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="131072"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="+Inf"} 5 +milvus_querynode_sq_segment_latency_sum{node_id="37",query_type="query",segment_state="Sealed"} 10 +milvus_querynode_sq_segment_latency_count{node_id="37",query_type="query",segment_state="Sealed"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1"} 1 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="64"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="128"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="256"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="512"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1024"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2048"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4096"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8192"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16384"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32768"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="65536"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="131072"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_sq_segment_latency_sum{node_id="37",query_type="search",segment_state="Sealed"} 3 +milvus_querynode_sq_segment_latency_count{node_id="37",query_type="search",segment_state="Sealed"} 2 # HELP milvus_querynode_sq_wait_tsafe_latency latency of search or query to wait for tsafe # TYPE milvus_querynode_sq_wait_tsafe_latency histogram -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="5"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="10"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="20"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="50"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="100"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="200"} 4 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="300"} 4 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="400"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="500"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="600"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="700"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="800"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="900"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="1000"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="1500"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="3000"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="10000"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="60000"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="query",le="+Inf"} 5 -milvus_querynode_sq_wait_tsafe_latency_sum{node_id="2",query_type="query"} 1122 -milvus_querynode_sq_wait_tsafe_latency_count{node_id="2",query_type="query"} 5 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="5"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="10"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="20"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="50"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="100"} 0 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="200"} 1 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="300"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="400"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="500"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="600"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="700"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="800"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="900"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="1000"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="1500"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="3000"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="10000"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="60000"} 2 -milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="2",query_type="search",le="+Inf"} 2 -milvus_querynode_sq_wait_tsafe_latency_sum{node_id="2",query_type="search"} 466 -milvus_querynode_sq_wait_tsafe_latency_count{node_id="2",query_type="search"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="2"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="4"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="8"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="16"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="32"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="64"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="128"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="256"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="37",query_type="query"} 1449 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="37",query_type="query"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="1"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="2"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="4"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="8"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="16"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="32"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="64"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="128"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="256"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="37",query_type="search"} 398 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="37",query_type="search"} 2 # HELP milvus_querynode_wait_processing_msg_count count of wait processing msg # TYPE milvus_querynode_wait_processing_msg_count gauge -milvus_querynode_wait_processing_msg_count{msg_type="delete",node_id="2"} 0 -milvus_querynode_wait_processing_msg_count{msg_type="insert",node_id="2"} 0 +milvus_querynode_wait_processing_msg_count{msg_type="delete",node_id="37"} 0 +milvus_querynode_wait_processing_msg_count{msg_type="insert",node_id="37"} 0 # HELP milvus_querynode_watch_dml_channel_latency latency of watch dml channel # TYPE milvus_querynode_watch_dml_channel_latency histogram -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="5"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="10"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="20"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="50"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="100"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="200"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="300"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="400"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="500"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="600"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="700"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="800"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="900"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="1000"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="1500"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="3000"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="10000"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="60000"} 1 -milvus_querynode_watch_dml_channel_latency_bucket{node_id="2",le="+Inf"} 1 -milvus_querynode_watch_dml_channel_latency_sum{node_id="2"} 0 -milvus_querynode_watch_dml_channel_latency_count{node_id="2"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="1"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="2"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="4"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="8"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="16"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="32"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="64"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="128"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="256"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="512"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="1024"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="2048"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="4096"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="8192"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="16384"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="32768"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="65536"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="131072"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_watch_dml_channel_latency_sum{node_id="37"} 0 +milvus_querynode_watch_dml_channel_latency_count{node_id="37"} 1 # HELP milvus_rootcoord_collection_num number of collections # TYPE milvus_rootcoord_collection_num gauge milvus_rootcoord_collection_num{db_name="default"} 0 @@ -2476,10 +2663,10 @@ milvus_rootcoord_credential_num 0 milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="success"} 1 milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="total"} 1 milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="fail"} 3 -milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="success"} 15 -milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="total"} 18 -milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="success"} 2 -milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="total"} 2 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="success"} 12 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="total"} 15 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="success"} 3 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="total"} 3 milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="success"} 1 milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="total"} 1 milvus_rootcoord_ddl_req_count{function_name="ListDatabases",status="success"} 2 @@ -2492,171 +2679,171 @@ milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="success"} milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="total"} 4 # HELP milvus_rootcoord_ddl_req_latency latency of each DDL operations # TYPE milvus_rootcoord_ddl_req_latency histogram -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="5"} 0 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="10"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="20"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="50"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="100"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="200"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="300"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="400"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="600"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="700"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="800"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="900"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="3000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="10000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="2"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="131072"} 1 milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="+Inf"} 1 -milvus_rootcoord_ddl_req_latency_sum{function_name="CreateCollection"} 7 +milvus_rootcoord_ddl_req_latency_sum{function_name="CreateCollection"} 4 milvus_rootcoord_ddl_req_latency_count{function_name="CreateCollection"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="5"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="10"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="20"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="50"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="100"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="200"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="300"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="400"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="500"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="600"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="700"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="800"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="900"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1000"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1500"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="3000"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="10000"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="60000"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="+Inf"} 15 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="2"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="4"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="8"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="16"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="32"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="64"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="128"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="256"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="512"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1024"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="2048"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="4096"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="8192"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="16384"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="32768"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="65536"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="131072"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="+Inf"} 12 milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeCollection"} 0 -milvus_rootcoord_ddl_req_latency_count{function_name="DescribeCollection"} 15 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="5"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="10"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="20"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="50"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="100"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="200"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="300"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="400"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="500"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="600"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="700"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="800"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="900"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1500"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="3000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="10000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="60000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="+Inf"} 2 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeCollection"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="2"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="4"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="8"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="16"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="32"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="64"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="128"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="256"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="512"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1024"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="2048"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="4096"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="8192"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="16384"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="32768"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="65536"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="131072"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="+Inf"} 3 milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeDatabase"} 0 -milvus_rootcoord_ddl_req_latency_count{function_name="DescribeDatabase"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="5"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="10"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="20"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="50"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="100"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="200"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="300"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="400"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="600"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="700"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="800"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="900"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="3000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="10000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeDatabase"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="2"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="131072"} 1 milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="+Inf"} 1 -milvus_rootcoord_ddl_req_latency_sum{function_name="DropCollection"} 0 +milvus_rootcoord_ddl_req_latency_sum{function_name="DropCollection"} 4 milvus_rootcoord_ddl_req_latency_count{function_name="DropCollection"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="5"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="10"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="20"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="50"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="100"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="200"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="300"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="400"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="500"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="600"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="700"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="800"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="900"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1500"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="3000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="10000"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="60000"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="2"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="4"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="8"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="16"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="32"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="64"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="128"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="256"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="512"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1024"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="2048"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="4096"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="8192"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="16384"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="32768"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="65536"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="131072"} 2 milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="+Inf"} 2 milvus_rootcoord_ddl_req_latency_sum{function_name="ListDatabases"} 0 milvus_rootcoord_ddl_req_latency_count{function_name="ListDatabases"} 2 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="5"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="10"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="20"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="50"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="100"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="200"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="300"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="400"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="600"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="700"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="800"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="900"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="3000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="10000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="2"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="131072"} 1 milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="+Inf"} 1 milvus_rootcoord_ddl_req_latency_sum{function_name="PolicyList"} 0 milvus_rootcoord_ddl_req_latency_count{function_name="PolicyList"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="5"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="10"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="20"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="50"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="100"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="200"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="300"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="400"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="600"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="700"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="800"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="900"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1500"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="3000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="10000"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="60000"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="2"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="131072"} 1 milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="+Inf"} 1 milvus_rootcoord_ddl_req_latency_sum{function_name="ShowCollections"} 0 milvus_rootcoord_ddl_req_latency_count{function_name="ShowCollections"} 1 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="5"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="10"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="20"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="50"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="100"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="200"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="300"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="400"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="500"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="600"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="700"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="800"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="900"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1000"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1500"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="3000"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="10000"} 4 -milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="60000"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="2"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="4"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="8"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="16"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="32"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="64"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="128"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="256"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="512"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1024"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="2048"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="4096"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="8192"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="16384"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="32768"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="65536"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="131072"} 4 milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="+Inf"} 4 milvus_rootcoord_ddl_req_latency_sum{function_name="ShowPartitions"} 0 milvus_rootcoord_ddl_req_latency_count{function_name="ShowPartitions"} 4 @@ -2676,20 +2863,20 @@ milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="+Inf"} 1 milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="CreateCollection"} 0 milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="CreateCollection"} 1 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.005"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.01"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.025"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.05"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.1"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.25"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.5"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="1"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="2.5"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="5"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="10"} 15 -milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="+Inf"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.005"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.01"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.025"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.05"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.1"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.25"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="1"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="2.5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="10"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="+Inf"} 12 milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="DescribeCollection"} 0 -milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DescribeCollection"} 15 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DescribeCollection"} 12 milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.005"} 1 milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.01"} 1 milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.025"} 1 @@ -2744,7 +2931,7 @@ milvus_rootcoord_entity_num{collection_name="hello_milvus",status="total"} 3001 milvus_rootcoord_force_deny_writing_counter 0 # HELP milvus_rootcoord_id_alloc_count count of ID allocated # TYPE milvus_rootcoord_id_alloc_count counter -milvus_rootcoord_id_alloc_count 400018 +milvus_rootcoord_id_alloc_count 400508 # HELP milvus_rootcoord_indexed_entity_num indexed number of entities, clustered by collection, index name and whether it's a vector index # TYPE milvus_rootcoord_indexed_entity_num gauge milvus_rootcoord_indexed_entity_num{collection_name="hello_milvus",index_name="embeddings",is_vector_index="true"} 3001 @@ -2759,7 +2946,17 @@ milvus_rootcoord_num_of_roles 0 milvus_rootcoord_partition_num 0 # HELP milvus_rootcoord_produce_tt_lag_ms now time minus tt per physical channel # TYPE milvus_rootcoord_produce_tt_lag_ms gauge -milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0"} 150 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_1"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10"} 137 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_2"} 87 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_3"} 94 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_4"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_5"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_6"} 95 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_7"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_8"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_9"} 97 # HELP milvus_rootcoord_proxy_num number of proxy nodes managered by rootcoord # TYPE milvus_rootcoord_proxy_num gauge milvus_rootcoord_proxy_num 1 @@ -2768,54 +2965,57 @@ milvus_rootcoord_proxy_num 1 milvus_rootcoord_qn_mem_high_water_level 0 # HELP milvus_rootcoord_sync_timetick_latency latency of synchronizing timetick message # TYPE milvus_rootcoord_sync_timetick_latency histogram -milvus_rootcoord_sync_timetick_latency_bucket{le="5"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="10"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="20"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="50"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="100"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="200"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="300"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="400"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="500"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="600"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="700"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="800"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="900"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="1000"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="1500"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="3000"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="10000"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="60000"} 51 -milvus_rootcoord_sync_timetick_latency_bucket{le="+Inf"} 51 -milvus_rootcoord_sync_timetick_latency_sum 0 -milvus_rootcoord_sync_timetick_latency_count 51 +milvus_rootcoord_sync_timetick_latency_bucket{le="1"} 42 +milvus_rootcoord_sync_timetick_latency_bucket{le="2"} 65 +milvus_rootcoord_sync_timetick_latency_bucket{le="4"} 181 +milvus_rootcoord_sync_timetick_latency_bucket{le="8"} 310 +milvus_rootcoord_sync_timetick_latency_bucket{le="16"} 349 +milvus_rootcoord_sync_timetick_latency_bucket{le="32"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="64"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="128"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="256"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="512"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="1024"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="2048"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="4096"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="8192"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="16384"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="32768"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="65536"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="131072"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="+Inf"} 353 +milvus_rootcoord_sync_timetick_latency_sum 1756 +milvus_rootcoord_sync_timetick_latency_count 353 # HELP milvus_rootcoord_timestamp lateste timestamp allocated in memory # TYPE milvus_rootcoord_timestamp gauge -milvus_rootcoord_timestamp 4.541724788559708e+17 +milvus_rootcoord_timestamp 4.548092784887726e+17 # HELP milvus_rootcoord_timestamp_saved timestamp saved in meta storage # TYPE milvus_rootcoord_timestamp_saved gauge -milvus_rootcoord_timestamp_saved 1.732530515e+09 +milvus_rootcoord_timestamp_saved 1.734959711e+09 # HELP milvus_runtime_info Runtime information of milvus # TYPE milvus_runtime_info gauge milvus_runtime_info{meta="etcd",mq="rocksmq"} 1 +# HELP milvus_thread_num the actual thread number of milvus process +# TYPE milvus_thread_num gauge +milvus_thread_num 73 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter -process_cpu_seconds_total 14.19 +process_cpu_seconds_total 8.06 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge -process_open_fds 70 +process_open_fds 71 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge -process_resident_memory_bytes 3.57658624e+08 +process_resident_memory_bytes 5.58592e+08 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge -process_start_time_seconds 1.73252970801e+09 +process_start_time_seconds 1.73495960802e+09 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge -process_virtual_memory_bytes 8.257282048e+09 +process_virtual_memory_bytes 8.522620928e+09 # HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. # TYPE process_virtual_memory_max_bytes gauge process_virtual_memory_max_bytes 1.8446744073709552e+19 @@ -2954,7 +3154,7 @@ build_latency_bucket{module="knowhere",le="32768"} 1 build_latency_bucket{module="knowhere",le="65536"} 1 build_latency_bucket{module="knowhere",le="1.048576e+06"} 1 build_latency_bucket{module="knowhere",le="+Inf"} 1 -build_latency_sum{module="knowhere"} 0.006943917 +build_latency_sum{module="knowhere"} 0.015760917 build_latency_count{module="knowhere"} 1 # HELP cache_hit_cnt cache hit cnt per request # TYPE cache_hit_cnt histogram @@ -3256,60 +3456,100 @@ hnsw_search_hops_sum{module="knowhere"} 0 hnsw_search_hops_count{module="knowhere"} 0 # HELP internal_core_search_latency [cpp]latency(us) of search on segment # TYPE internal_core_search_latency histogram -internal_core_search_latency_bucket{type="scalar_latency",le="1"} 0 -internal_core_search_latency_bucket{type="scalar_latency",le="2"} 0 -internal_core_search_latency_bucket{type="scalar_latency",le="4"} 0 -internal_core_search_latency_bucket{type="scalar_latency",le="8"} 1 -internal_core_search_latency_bucket{type="scalar_latency",le="16"} 1 -internal_core_search_latency_bucket{type="scalar_latency",le="32"} 1 -internal_core_search_latency_bucket{type="scalar_latency",le="64"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="128"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="256"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="512"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="1024"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="2048"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="4096"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="8192"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="16384"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="32768"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="65536"} 2 -internal_core_search_latency_bucket{type="scalar_latency",le="+Inf"} 2 -internal_core_search_latency_sum{type="scalar_latency"} 57.667 -internal_core_search_latency_count{type="scalar_latency"} 2 +internal_core_search_latency_bucket{type="groupby_latency",le="1"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="2"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="4"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="8"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="16"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="32"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="64"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="128"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="256"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="512"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="1024"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="2048"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="4096"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="8192"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="16384"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="32768"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="65536"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="+Inf"} 0 +internal_core_search_latency_sum{type="groupby_latency"} 0 +internal_core_search_latency_count{type="groupby_latency"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="1"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="2"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="4"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="8"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="16"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="32"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="64"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="128"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="256"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="512"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="1024"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="2048"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="4096"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="8192"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="16384"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="32768"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="65536"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="+Inf"} 0 +internal_core_search_latency_sum{type="iterative_filter_latency"} 0 +internal_core_search_latency_count{type="iterative_filter_latency"} 0 +internal_core_search_latency_bucket{type="scalar_latency",le="1"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="2"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="4"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="8"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="16"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="32"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="64"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="128"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="256"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="512"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="1024"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="2048"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="4096"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="8192"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="16384"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="32768"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="65536"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="+Inf"} 6 +internal_core_search_latency_sum{type="scalar_latency"} 0.761541 +internal_core_search_latency_count{type="scalar_latency"} 6 internal_core_search_latency_bucket{type="scalar_proportion",le="0"} 0 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.05"} 1 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.1"} 1 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.15"} 1 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.2"} 1 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.25"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.3"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.35"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.4"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.45"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.5"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.55"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.6"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.65"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.7"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.75"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.8"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.85"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.9"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="0.95"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="1"} 2 -internal_core_search_latency_bucket{type="scalar_proportion",le="+Inf"} 2 -internal_core_search_latency_sum{type="scalar_proportion"} 0.2503988069921382 -internal_core_search_latency_count{type="scalar_proportion"} 2 -internal_core_search_latency_bucket{type="vector_latency",le="1"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="2"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="4"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="8"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="16"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="32"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="64"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="128"} 0 -internal_core_search_latency_bucket{type="vector_latency",le="256"} 1 -internal_core_search_latency_bucket{type="vector_latency",le="512"} 1 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.05"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.1"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.15"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.2"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.25"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.3"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.35"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.4"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.45"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.5"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.55"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.6"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.65"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.7"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.75"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.8"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.85"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.9"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.95"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="1"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="+Inf"} 0 +internal_core_search_latency_sum{type="scalar_proportion"} 0 +internal_core_search_latency_count{type="scalar_proportion"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="1"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="2"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="4"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="8"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="16"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="32"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="64"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="128"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="256"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="512"} 2 internal_core_search_latency_bucket{type="vector_latency",le="1024"} 2 internal_core_search_latency_bucket{type="vector_latency",le="2048"} 2 internal_core_search_latency_bucket{type="vector_latency",le="4096"} 2 @@ -3318,27 +3558,27 @@ internal_core_search_latency_bucket{type="vector_latency",le="16384"} 2 internal_core_search_latency_bucket{type="vector_latency",le="32768"} 2 internal_core_search_latency_bucket{type="vector_latency",le="65536"} 2 internal_core_search_latency_bucket{type="vector_latency",le="+Inf"} 2 -internal_core_search_latency_sum{type="vector_latency"} 986.25 +internal_core_search_latency_sum{type="vector_latency"} 1.549042 internal_core_search_latency_count{type="vector_latency"} 2 # HELP internal_mmap_allocated_space_bytes [cpp]mmap allocated space stats # TYPE internal_mmap_allocated_space_bytes histogram internal_mmap_allocated_space_bytes_bucket{type="anon",le="1024"} 0 internal_mmap_allocated_space_bytes_bucket{type="anon",le="8192"} 0 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="65536"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="262144"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="524288"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.048576e+06"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="4.194304e+06"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="8.388608e+06"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.6777216e+07"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="6.7108864e+07"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.34217728e+08"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="2.68435456e+08"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="5.36870912e+08"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.073741824e+09"} 2 -internal_mmap_allocated_space_bytes_bucket{type="anon",le="+Inf"} 2 -internal_mmap_allocated_space_bytes_sum{type="anon"} 46908 -internal_mmap_allocated_space_bytes_count{type="anon"} 2 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="65536"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="262144"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="524288"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.048576e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="4.194304e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="8.388608e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.6777216e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="6.7108864e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.34217728e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="2.68435456e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="5.36870912e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.073741824e+09"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="+Inf"} 0 +internal_mmap_allocated_space_bytes_sum{type="anon"} 0 +internal_mmap_allocated_space_bytes_count{type="anon"} 0 internal_mmap_allocated_space_bytes_bucket{type="file",le="1024"} 0 internal_mmap_allocated_space_bytes_bucket{type="file",le="8192"} 0 internal_mmap_allocated_space_bytes_bucket{type="file",le="65536"} 0 @@ -3356,6 +3596,10 @@ internal_mmap_allocated_space_bytes_bucket{type="file",le="1.073741824e+09"} 0 internal_mmap_allocated_space_bytes_bucket{type="file",le="+Inf"} 0 internal_mmap_allocated_space_bytes_sum{type="file"} 0 internal_mmap_allocated_space_bytes_count{type="file"} 0 +# HELP internal_mmap_in_used_count [cpp]mmap in used count stats +# TYPE internal_mmap_in_used_count gauge +internal_mmap_in_used_count{type="anon"} 0 +internal_mmap_in_used_count{type="file"} 0 # HELP internal_mmap_in_used_space_bytes [cpp]mmap in used space stats # TYPE internal_mmap_in_used_space_bytes gauge internal_mmap_in_used_space_bytes{type="anon"} 0 @@ -3666,7 +3910,7 @@ load_latency_bucket{module="knowhere",le="32768"} 1 load_latency_bucket{module="knowhere",le="65536"} 1 load_latency_bucket{module="knowhere",le="1.048576e+06"} 1 load_latency_bucket{module="knowhere",le="+Inf"} 1 -load_latency_sum{module="knowhere"} 0.5315420000000001 +load_latency_sum{module="knowhere"} 0.1285 load_latency_count{module="knowhere"} 1 # HELP quant_compute_cnt quant compute cnt per request # TYPE quant_compute_cnt histogram @@ -3846,7 +4090,7 @@ search_latency_bucket{module="knowhere",le="32768"} 2 search_latency_bucket{module="knowhere",le="65536"} 2 search_latency_bucket{module="knowhere",le="1.048576e+06"} 2 search_latency_bucket{module="knowhere",le="+Inf"} 2 -search_latency_sum{module="knowhere"} 0.759957 +search_latency_sum{module="knowhere"} 0.8826250000000001 search_latency_count{module="knowhere"} 2 # HELP search_topk search topk # TYPE search_topk histogram diff --git a/milvus/tests/test_e2e.py b/milvus/tests/test_e2e.py index a6ccb89fdb8dd..058f4713b4cdf 100644 --- a/milvus/tests/test_e2e.py +++ b/milvus/tests/test_e2e.py @@ -13,7 +13,10 @@ def test_check_milvus_e2e(dd_agent_check, instance): aggregator = dd_agent_check(instance, rate=True) for metric, _ in common.STANDALONE_TEST_METRICS.items(): - if metric == 'milvus.datacoord.import_tasks': # this metric needs a more complex setup to appear + if metric in [ + 'milvus.datacoord.import_tasks', + 'milvus.datacoord.index.task', + ]: # these metrics need a more complex setup to appear continue aggregator.assert_metric(name=metric) From e2e1b5fb832699e27fce4c023304011986704b23 Mon Sep 17 00:00:00 2001 From: David Kirov Date: Mon, 23 Dec 2024 18:15:14 +0200 Subject: [PATCH 31/31] Improve README --- milvus/README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/milvus/README.md b/milvus/README.md index 85f293f5c2cfb..e9278fa0871f9 100644 --- a/milvus/README.md +++ b/milvus/README.md @@ -6,8 +6,6 @@ This check monitors [Milvus][1] through the Datadog Agent. It provides insights ## Setup -Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. - ### Installation The Milvus check is included in the [Datadog Agent][2] package. @@ -15,10 +13,27 @@ No additional installation is needed on your server. ### Configuration +#### Host + +Follow the instructions below to configure this check for an Agent running on a host. For containerized environments, see the [Containerized](#containerized) section. + + + + 1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your Milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options. 2. [Restart the Agent][5]. + + + +#### Containerized + +For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. + + + + ### Validation [Run the Agent's status subcommand][6] and look for `milvus` under the Checks section.