[AI-4451] Add Milvus integration (#19055)

* Add Milvus skeleton

* Add metrics to metadata.csv

* Add periods in metadata.csv

* Update spec

* Update config models

* Update manifest

* Update check.py

* Add process signature

* Add units in metadata.csv

* Remove process signature

* Add payload fixture

* Add tests

* Fix metadata.csv

* Add non-Milvus specific metrics

* Update metrics map

* Expand histogram metrics

* Fix metrics mapping and divide test metrics

* Passing unit tests

* Add e2e env and test

* Run lint

* Fix CI

* Switch to deprecated classifier tags for CI

* Sync labeler

* Remove dashboard and add metric check

* Implement review suggestions

* Incorporate documentation suggestions

* Capitalize unix in metadata description

Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com>

* Incorporate documentation suggestions

* Move metrics out of standalone test

* Fix unit tests

* Improve README

---------

Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com>

.codecov.yml

@@ -410,6 +410,10 @@ coverage:
         target: 75
         flags:
         - mesos_master
+      Milvus:
+        target: 75
+        flags:
+        - milvus
       MongoDB:
         target: 75
         flags:
@@ -1262,6 +1266,11 @@ flags:
     paths:
     - mesos_slave/datadog_checks/mesos_slave
     - mesos_slave/tests
+  milvus:
+    carryforward: true
+    paths:
+    - milvus/datadog_checks/milvus
+    - milvus/tests
   mongo:
     carryforward: true
     paths:

.github/workflows/config/labeler.yml

@@ -353,6 +353,8 @@ integration/mesos_slave:
 - mesos_slave/**/*
 integration/metabase:
 - metabase/**/*
+integration/milvus:
+- milvus/**/*
 integration/mimecast:
 - mimecast/**/*
 integration/mongo:

.github/workflows/test-all.yml

@@ -2494,6 +2494,26 @@ jobs:
       minimum-base-package: ${{ inputs.minimum-base-package }}
       pytest-args: ${{ inputs.pytest-args }}
     secrets: inherit
+  j062aeb0:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Milvus
+      target: milvus
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+      pytest-args: ${{ inputs.pytest-args }}
+    secrets: inherit
   j91231ff:
     uses: ./.github/workflows/test-target.yml
     with:

milvus/CHANGELOG.md

@@ -0,0 +1,4 @@
+# CHANGELOG - Milvus
+
+<!-- towncrier release notes start -->
+

milvus/README.md

@@ -0,0 +1,68 @@
+# Agent Check: Milvus
+
+## Overview
+
+This check monitors [Milvus][1] through the Datadog Agent. It provides insights into your Milvus deployment's performance by collecting information about the latency and number of executions of individual operations. This integration also allows for monitoring the size and ressource allocation of your deployment.
+
+## Setup
+
+### Installation
+
+The Milvus check is included in the [Datadog Agent][2] package.
+No additional installation is needed on your server.
+
+### Configuration
+
+#### Host
+
+Follow the instructions below to configure this check for an Agent running on a host. For containerized environments, see the [Containerized](#containerized) section.
+
+<!-- xxx tabs xxx -->
+<!-- xxx tab "Host" xxx -->
+
+1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your Milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options.
+
+2. [Restart the Agent][5].
+
+<!-- xxz tab xxx -->
+<!-- xxx tab "Containerized" xxx -->
+
+#### Containerized
+
+For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions.
+
+<!-- xxz tab xxx -->
+<!-- xxz tabs xxx -->
+
+### Validation
+
+[Run the Agent's status subcommand][6] and look for `milvus` under the Checks section.
+
+## Data Collected
+
+### Metrics
+
+See [metadata.csv][7] for a list of metrics provided by this integration.
+
+### Events
+
+The Milvus integration does not include any events.
+
+### Service Checks
+
+The Milvus integration does not include any service checks.
+
+## Troubleshooting
+
+Need help? Contact [Datadog support][9].
+
+
+[1]: https://milvus.io/
+[2]: https://app.datadoghq.com/account/settings/agent/latest
+[3]: https://docs.datadoghq.com/agent/kubernetes/integrations/
+[4]: https://github.com/DataDog/integrations-core/blob/master/milvus/datadog_checks/milvus/data/conf.yaml.example
+[5]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent
+[6]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information
+[7]: https://github.com/DataDog/integrations-core/blob/master/milvus/metadata.csv
+[8]: https://github.com/DataDog/integrations-core/blob/master/milvus/assets/service_checks.json
+[9]: https://docs.datadoghq.com/help/

milvus/assets/configuration/spec.yaml

@@ -0,0 +1,15 @@
+name: Milvus
+files:
+- name: milvus.yaml
+  options:
+    - template: init_config
+      options:
+        - template: init_config/openmetrics
+    - template: instances
+      options:
+        - template: instances/openmetrics
+          overrides:
+            openmetrics_endpoint.value.example: http://localhost:9091/metrics
+            openmetrics_endpoint.description: |
+              Endpoint exposing Milvus' Prometheus metrics. For more information, refer to 
+              https://milvus.io/docs/monitor.md#Monitor-metrics-with-Prometheus.

milvus/assets/service_checks.json

@@ -0,0 +1 @@
+[]

milvus/changelog.d/19055.added

@@ -0,0 +1 @@
+Initial Release

milvus/datadog_checks/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

milvus/datadog_checks/milvus/__about__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__version__ = '0.0.1'

milvus/datadog_checks/milvus/__init__.py

@@ -0,0 +1,7 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from .__about__ import __version__
+from .check import MilvusCheck
+
+__all__ = ['__version__', 'MilvusCheck']

milvus/datadog_checks/milvus/check.py

@@ -0,0 +1,21 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+from datadog_checks.base import OpenMetricsBaseCheckV2
+from datadog_checks.milvus.metrics import METRIC_MAP, RENAME_LABELS_MAP
+
+
+class MilvusCheck(OpenMetricsBaseCheckV2):
+    __NAMESPACE__ = 'milvus'
+
+    DEFAULT_METRIC_LIMIT = 0
+
+    def __init__(self, name, init_config, instances=None):
+        super(MilvusCheck, self).__init__(name, init_config, instances)
+
+    def get_default_config(self):
+        return {
+            'metrics': [METRIC_MAP],
+            'rename_labels': RENAME_LABELS_MAP,
+        }

milvus/datadog_checks/milvus/config_models/__init__.py

@@ -0,0 +1,24 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+from .instance import InstanceConfig
+from .shared import SharedConfig
+
+
+class ConfigMixin:
+    _config_model_instance: InstanceConfig
+    _config_model_shared: SharedConfig
+
+    @property
+    def config(self) -> InstanceConfig:
+        return self._config_model_instance
+
+    @property
+    def shared_config(self) -> SharedConfig:
+        return self._config_model_shared

milvus/datadog_checks/milvus/config_models/defaults.py

@@ -0,0 +1,132 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+
+def shared_skip_proxy():
+    return False
+
+
+def shared_timeout():
+    return 10
+
+
+def instance_allow_redirects():
+    return True
+
+
+def instance_auth_type():
+    return 'basic'
+
+
+def instance_cache_metric_wildcards():
+    return True
+
+
+def instance_cache_shared_labels():
+    return True
+
+
+def instance_collect_counters_with_distributions():
+    return False
+
+
+def instance_collect_histogram_buckets():
+    return True
+
+
+def instance_disable_generic_tags():
+    return False
+
+
+def instance_empty_default_hostname():
+    return False
+
+
+def instance_enable_health_service_check():
+    return True
+
+
+def instance_histogram_buckets_as_distributions():
+    return False
+
+
+def instance_ignore_connection_errors():
+    return False
+
+
+def instance_kerberos_auth():
+    return 'disabled'
+
+
+def instance_kerberos_delegate():
+    return False
+
+
+def instance_kerberos_force_initiate():
+    return False
+
+
+def instance_log_requests():
+    return False
+
+
+def instance_min_collection_interval():
+    return 15
+
+
+def instance_non_cumulative_histogram_buckets():
+    return False
+
+
+def instance_persist_connections():
+    return False
+
+
+def instance_request_size():
+    return 16
+
+
+def instance_skip_proxy():
+    return False
+
+
+def instance_tag_by_endpoint():
+    return True
+
+
+def instance_telemetry():
+    return False
+
+
+def instance_timeout():
+    return 10
+
+
+def instance_tls_ignore_warning():
+    return False
+
+
+def instance_tls_use_host_header():
+    return False
+
+
+def instance_tls_verify():
+    return True
+
+
+def instance_use_latest_spec():
+    return False
+
+
+def instance_use_legacy_auth_encoding():
+    return True
+
+
+def instance_use_process_start_time():
+    return False