diff --git a/.codecov.yml b/.codecov.yml index 12d18bf18d059..9aef58bee0c1c 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -410,6 +410,10 @@ coverage: target: 75 flags: - mesos_master + Milvus: + target: 75 + flags: + - milvus MongoDB: target: 75 flags: @@ -1254,6 +1258,11 @@ flags: paths: - mesos_slave/datadog_checks/mesos_slave - mesos_slave/tests + milvus: + carryforward: true + paths: + - milvus/datadog_checks/milvus + - milvus/tests mongo: carryforward: true paths: diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 4fe0c7e782c20..44a07458a0107 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -339,6 +339,8 @@ integration/mesos_slave: - mesos_slave/**/* integration/metabase: - metabase/**/* +integration/milvus: +- milvus/**/* integration/mimecast: - mimecast/**/* integration/mongo: diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 53fc9b5f077c9..b32d88a9dc0c6 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -2494,6 +2494,26 @@ jobs: minimum-base-package: ${{ inputs.minimum-base-package }} pytest-args: ${{ inputs.pytest-args }} secrets: inherit + j062aeb0: + uses: ./.github/workflows/test-target.yml + with: + job-name: Milvus + target: milvus + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + python-version: "${{ inputs.python-version }}" + standard: ${{ inputs.standard }} + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + minimum-base-package: ${{ inputs.minimum-base-package }} + pytest-args: ${{ inputs.pytest-args }} + secrets: inherit j91231ff: uses: ./.github/workflows/test-target.yml with: diff --git a/milvus/CHANGELOG.md b/milvus/CHANGELOG.md new file mode 100644 index 0000000000000..dd71dbd975e36 --- /dev/null +++ b/milvus/CHANGELOG.md @@ -0,0 +1,4 @@ +# CHANGELOG - Milvus + + + diff --git a/milvus/README.md b/milvus/README.md new file mode 100644 index 0000000000000..e9278fa0871f9 --- /dev/null +++ b/milvus/README.md @@ -0,0 +1,68 @@ +# Agent Check: Milvus + +## Overview + +This check monitors [Milvus][1] through the Datadog Agent. It provides insights into your Milvus deployment's performance by collecting information about the latency and number of executions of individual operations. This integration also allows for monitoring the size and ressource allocation of your deployment. + +## Setup + +### Installation + +The Milvus check is included in the [Datadog Agent][2] package. +No additional installation is needed on your server. + +### Configuration + +#### Host + +Follow the instructions below to configure this check for an Agent running on a host. For containerized environments, see the [Containerized](#containerized) section. + + + + +1. Edit the `milvus.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your Milvus performance data. See the [sample milvus.d/conf.yaml][4] for all available configuration options. + +2. [Restart the Agent][5]. + + + + +#### Containerized + +For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. + + + + +### Validation + +[Run the Agent's status subcommand][6] and look for `milvus` under the Checks section. + +## Data Collected + +### Metrics + +See [metadata.csv][7] for a list of metrics provided by this integration. + +### Events + +The Milvus integration does not include any events. + +### Service Checks + +The Milvus integration does not include any service checks. + +## Troubleshooting + +Need help? Contact [Datadog support][9]. + + +[1]: https://milvus.io/ +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/agent/kubernetes/integrations/ +[4]: https://github.com/DataDog/integrations-core/blob/master/milvus/datadog_checks/milvus/data/conf.yaml.example +[5]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent +[6]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[7]: https://github.com/DataDog/integrations-core/blob/master/milvus/metadata.csv +[8]: https://github.com/DataDog/integrations-core/blob/master/milvus/assets/service_checks.json +[9]: https://docs.datadoghq.com/help/ diff --git a/milvus/assets/configuration/spec.yaml b/milvus/assets/configuration/spec.yaml new file mode 100644 index 0000000000000..100b70fcc547f --- /dev/null +++ b/milvus/assets/configuration/spec.yaml @@ -0,0 +1,15 @@ +name: Milvus +files: +- name: milvus.yaml + options: + - template: init_config + options: + - template: init_config/openmetrics + - template: instances + options: + - template: instances/openmetrics + overrides: + openmetrics_endpoint.value.example: http://localhost:9091/metrics + openmetrics_endpoint.description: | + Endpoint exposing Milvus' Prometheus metrics. For more information, refer to + https://milvus.io/docs/monitor.md#Monitor-metrics-with-Prometheus. diff --git a/milvus/assets/service_checks.json b/milvus/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/milvus/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/milvus/changelog.d/19055.added b/milvus/changelog.d/19055.added new file mode 100644 index 0000000000000..aa949b47b7b41 --- /dev/null +++ b/milvus/changelog.d/19055.added @@ -0,0 +1 @@ +Initial Release \ No newline at end of file diff --git a/milvus/datadog_checks/__init__.py b/milvus/datadog_checks/__init__.py new file mode 100644 index 0000000000000..1517d901c0aae --- /dev/null +++ b/milvus/datadog_checks/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/milvus/datadog_checks/milvus/__about__.py b/milvus/datadog_checks/milvus/__about__.py new file mode 100644 index 0000000000000..e9541ce83e9e5 --- /dev/null +++ b/milvus/datadog_checks/milvus/__about__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__version__ = '0.0.1' diff --git a/milvus/datadog_checks/milvus/__init__.py b/milvus/datadog_checks/milvus/__init__.py new file mode 100644 index 0000000000000..b0126262a42a2 --- /dev/null +++ b/milvus/datadog_checks/milvus/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .__about__ import __version__ +from .check import MilvusCheck + +__all__ = ['__version__', 'MilvusCheck'] diff --git a/milvus/datadog_checks/milvus/check.py b/milvus/datadog_checks/milvus/check.py new file mode 100644 index 0000000000000..d1be0f56ca38e --- /dev/null +++ b/milvus/datadog_checks/milvus/check.py @@ -0,0 +1,21 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from datadog_checks.base import OpenMetricsBaseCheckV2 +from datadog_checks.milvus.metrics import METRIC_MAP, RENAME_LABELS_MAP + + +class MilvusCheck(OpenMetricsBaseCheckV2): + __NAMESPACE__ = 'milvus' + + DEFAULT_METRIC_LIMIT = 0 + + def __init__(self, name, init_config, instances=None): + super(MilvusCheck, self).__init__(name, init_config, instances) + + def get_default_config(self): + return { + 'metrics': [METRIC_MAP], + 'rename_labels': RENAME_LABELS_MAP, + } diff --git a/milvus/datadog_checks/milvus/config_models/__init__.py b/milvus/datadog_checks/milvus/config_models/__init__.py new file mode 100644 index 0000000000000..106fff2032f68 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/__init__.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/milvus/datadog_checks/milvus/config_models/defaults.py b/milvus/datadog_checks/milvus/config_models/defaults.py new file mode 100644 index 0000000000000..bf7519af75f42 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/defaults.py @@ -0,0 +1,132 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +def shared_skip_proxy(): + return False + + +def shared_timeout(): + return 10 + + +def instance_allow_redirects(): + return True + + +def instance_auth_type(): + return 'basic' + + +def instance_cache_metric_wildcards(): + return True + + +def instance_cache_shared_labels(): + return True + + +def instance_collect_counters_with_distributions(): + return False + + +def instance_collect_histogram_buckets(): + return True + + +def instance_disable_generic_tags(): + return False + + +def instance_empty_default_hostname(): + return False + + +def instance_enable_health_service_check(): + return True + + +def instance_histogram_buckets_as_distributions(): + return False + + +def instance_ignore_connection_errors(): + return False + + +def instance_kerberos_auth(): + return 'disabled' + + +def instance_kerberos_delegate(): + return False + + +def instance_kerberos_force_initiate(): + return False + + +def instance_log_requests(): + return False + + +def instance_min_collection_interval(): + return 15 + + +def instance_non_cumulative_histogram_buckets(): + return False + + +def instance_persist_connections(): + return False + + +def instance_request_size(): + return 16 + + +def instance_skip_proxy(): + return False + + +def instance_tag_by_endpoint(): + return True + + +def instance_telemetry(): + return False + + +def instance_timeout(): + return 10 + + +def instance_tls_ignore_warning(): + return False + + +def instance_tls_use_host_header(): + return False + + +def instance_tls_verify(): + return True + + +def instance_use_latest_spec(): + return False + + +def instance_use_legacy_auth_encoding(): + return True + + +def instance_use_process_start_time(): + return False diff --git a/milvus/datadog_checks/milvus/config_models/instance.py b/milvus/datadog_checks/milvus/config_models/instance.py new file mode 100644 index 0000000000000..8e39a0e921719 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/instance.py @@ -0,0 +1,171 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from types import MappingProxyType +from typing import Any, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class AuthToken(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + reader: Optional[MappingProxyType[str, Any]] = None + writer: Optional[MappingProxyType[str, Any]] = None + + +class ExtraMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra='allow', + frozen=True, + ) + name: Optional[str] = None + type: Optional[str] = None + + +class MetricPatterns(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + + +class Metrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra='allow', + frozen=True, + ) + name: Optional[str] = None + type: Optional[str] = None + + +class Proxy(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + http: Optional[str] = None + https: Optional[str] = None + no_proxy: Optional[tuple[str, ...]] = None + + +class ShareLabels(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + labels: Optional[tuple[str, ...]] = None + match: Optional[tuple[str, ...]] = None + + +class InstanceConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + allow_redirects: Optional[bool] = None + auth_token: Optional[AuthToken] = None + auth_type: Optional[str] = None + aws_host: Optional[str] = None + aws_region: Optional[str] = None + aws_service: Optional[str] = None + cache_metric_wildcards: Optional[bool] = None + cache_shared_labels: Optional[bool] = None + collect_counters_with_distributions: Optional[bool] = None + collect_histogram_buckets: Optional[bool] = None + connect_timeout: Optional[float] = None + disable_generic_tags: Optional[bool] = None + empty_default_hostname: Optional[bool] = None + enable_health_service_check: Optional[bool] = None + exclude_labels: Optional[tuple[str, ...]] = None + exclude_metrics: Optional[tuple[str, ...]] = None + exclude_metrics_by_labels: Optional[MappingProxyType[str, Union[bool, tuple[str, ...]]]] = None + extra_headers: Optional[MappingProxyType[str, Any]] = None + extra_metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, ExtraMetrics]]], ...]] = None + headers: Optional[MappingProxyType[str, Any]] = None + histogram_buckets_as_distributions: Optional[bool] = None + hostname_format: Optional[str] = None + hostname_label: Optional[str] = None + ignore_connection_errors: Optional[bool] = None + ignore_tags: Optional[tuple[str, ...]] = None + include_labels: Optional[tuple[str, ...]] = None + kerberos_auth: Optional[str] = None + kerberos_cache: Optional[str] = None + kerberos_delegate: Optional[bool] = None + kerberos_force_initiate: Optional[bool] = None + kerberos_hostname: Optional[str] = None + kerberos_keytab: Optional[str] = None + kerberos_principal: Optional[str] = None + log_requests: Optional[bool] = None + metric_patterns: Optional[MetricPatterns] = None + metrics: Optional[tuple[Union[str, MappingProxyType[str, Union[str, Metrics]]], ...]] = None + min_collection_interval: Optional[float] = None + namespace: Optional[str] = Field(None, pattern='\\w*') + non_cumulative_histogram_buckets: Optional[bool] = None + ntlm_domain: Optional[str] = None + openmetrics_endpoint: str + password: Optional[str] = None + persist_connections: Optional[bool] = None + proxy: Optional[Proxy] = None + raw_line_filters: Optional[tuple[str, ...]] = None + raw_metric_prefix: Optional[str] = None + read_timeout: Optional[float] = None + rename_labels: Optional[MappingProxyType[str, Any]] = None + request_size: Optional[float] = None + service: Optional[str] = None + share_labels: Optional[MappingProxyType[str, Union[bool, ShareLabels]]] = None + skip_proxy: Optional[bool] = None + tag_by_endpoint: Optional[bool] = None + tags: Optional[tuple[str, ...]] = None + telemetry: Optional[bool] = None + timeout: Optional[float] = None + tls_ca_cert: Optional[str] = None + tls_cert: Optional[str] = None + tls_ignore_warning: Optional[bool] = None + tls_private_key: Optional[str] = None + tls_protocols_allowed: Optional[tuple[str, ...]] = None + tls_use_host_header: Optional[bool] = None + tls_verify: Optional[bool] = None + use_latest_spec: Optional[bool] = None + use_legacy_auth_encoding: Optional[bool] = None + use_process_start_time: Optional[bool] = None + username: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) diff --git a/milvus/datadog_checks/milvus/config_models/shared.py b/milvus/datadog_checks/milvus/config_models/shared.py new file mode 100644 index 0000000000000..0e8a9ecab10a2 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/shared.py @@ -0,0 +1,60 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class Proxy(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + http: Optional[str] = None + https: Optional[str] = None + no_proxy: Optional[tuple[str, ...]] = None + + +class SharedConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + proxy: Optional[Proxy] = None + service: Optional[str] = None + skip_proxy: Optional[bool] = None + timeout: Optional[float] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'shared_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) diff --git a/milvus/datadog_checks/milvus/config_models/validators.py b/milvus/datadog_checks/milvus/config_models/validators.py new file mode 100644 index 0000000000000..1a91a82658202 --- /dev/null +++ b/milvus/datadog_checks/milvus/config_models/validators.py @@ -0,0 +1,40 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from urllib.parse import urlparse + + +def initialize_instance(values, **kwargs): + if 'openmetrics_endpoint' in values: + validate_url( + values['openmetrics_endpoint'], + required_path='/metrics', + example='http://localhost:9091/metrics', + config='openmetrics_endpoint', + ) + + return values + + +def validate_url(url, required_path=None, example=None, config=None): + # Validates the endpoint to ensure that the components are present. + # For OpenMetrics: Scheme, netloc and /metrics path + # For API: Scheme, netloc and no path + # Logs out troubleshooting errors and example + url_parsed = urlparse(url) + errors = [] + + if not url_parsed.netloc: + errors.append("couldn't properly parse endpoint") + if not url_parsed.scheme: + errors.append("http or https scheme is missing") + if required_path and url_parsed.path != required_path: + errors.append(f"URL should end with {required_path}") + if not required_path and url_parsed.path: + errors.append("should not contain a path or trailing /") + + if errors: + error_message = ", ".join(errors) + raise ValueError( + f"{config}: {url} is incorrectly configured. Errors detected: {error_message}. Example: {example}" + ) diff --git a/milvus/datadog_checks/milvus/data/conf.yaml.example b/milvus/datadog_checks/milvus/data/conf.yaml.example new file mode 100644 index 0000000000000..b275903aba352 --- /dev/null +++ b/milvus/datadog_checks/milvus/data/conf.yaml.example @@ -0,0 +1,626 @@ +## All options defined here are available to all instances. +# +init_config: + + ## @param proxy - mapping - optional + ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported like so: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## If set to `true`, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + + ## @param timeout - number - optional - default: 10 + ## The timeout for connecting to services. + # + # timeout: 10 + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# +instances: + + ## @param openmetrics_endpoint - string - required + ## Endpoint exposing Milvus' Prometheus metrics. For more information, refer to + ## https://milvus.io/docs/monitor.md#Monitor-metrics-with-Prometheus. + # + - openmetrics_endpoint: http://localhost:9091/metrics + + ## @param raw_metric_prefix - string - optional + ## A prefix that is removed from all exposed metric names, if present. + ## All configuration options will use the prefix-less name. + # + # raw_metric_prefix: _ + + ## @param extra_metrics - (list of string or mapping) - optional + ## This list defines metrics to collect from the `openmetrics_endpoint`, in addition to + ## what the check collects by default. If the check already collects a metric, then + ## metric definitions here take precedence. Metrics may be defined in 3 ways: + ## + ## 1. If the item is a string, then it represents the exposed metric name, and + ## the sent metric name will be identical. For example: + ## + ## extra_metrics: + ## - + ## - + ## 2. If the item is a mapping, then the keys represent the exposed metric names. + ## + ## a. If a value is a string, then it represents the sent metric name. For example: + ## + ## extra_metrics: + ## - : + ## - : + ## b. If a value is a mapping, then it must have a `name` and/or `type` key. + ## The `name` represents the sent metric name, and the `type` represents how + ## the metric should be handled, overriding any type information the endpoint + ## may provide. For example: + ## + ## extra_metrics: + ## - : + ## name: + ## type: + ## - : + ## name: + ## type: + ## + ## The supported native types are `gauge`, `counter`, `histogram`, and `summary`. + ## + ## Note: To collect counter metrics with names ending in `_total`, specify the metric name without the `_total` + ## suffix. For example, to collect the counter metric `promhttp_metric_handler_requests_total`, specify + ## `promhttp_metric_handler_requests`. This submits to Datadog the metric name appended with `.count`. + ## For more information, see: + ## https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes + ## + ## Regular expressions may be used to match the exposed metric names, for example: + ## + ## extra_metrics: + ## - ^network_(ingress|egress)_.+ + ## - .+: + ## type: gauge + # + # extra_metrics: [] + + ## @param exclude_metrics - list of strings - optional + ## A list of metrics to exclude, with each entry being either + ## the exact metric name or a regular expression. + ## In order to exclude all metrics but the ones matching a specific filter, + ## you can use a negative lookahead regex like: + ## - ^(?!foo).*$ + # + # exclude_metrics: [] + + ## @param exclude_metrics_by_labels - mapping - optional + ## A mapping of labels to exclude metrics with matching label name and their corresponding metric values. To match + ## all values of a label, set it to `true`. + ## + ## Note: Label filtering happens before `rename_labels`. + ## + ## For example, the following configuration instructs the check to exclude all metrics with + ## a label `worker` or a label `pid` with the value of either `23` or `42`. + ## + ## exclude_metrics_by_labels: + ## worker: true + ## pid: + ## - '23' + ## - '42' + # + # exclude_metrics_by_labels: {} + + ## @param exclude_labels - list of strings - optional + ## A list of labels to exclude, useful for high cardinality values like timestamps or UUIDs. + ## May be used in conjunction with `include_labels`. + ## Labels defined in `exclude_labels` will take precedence in case of overlap. + ## + ## Note: Label filtering happens before `rename_labels`. + # + # exclude_labels: [] + + ## @param include_labels - list of strings - optional + ## A list of labels to include. May be used in conjunction with `exclude_labels`. + ## Labels defined in `exclude_labels` will take precedence in case of overlap. + ## + ## Note: Label filtering happens before `rename_labels`. + # + # include_labels: [] + + ## @param rename_labels - mapping - optional + ## A mapping of label names to their new names. + # + # rename_labels: + # : + # : + + ## @param enable_health_service_check - boolean - optional - default: true + ## Whether or not to send a service check named `.openmetrics.health` which reports + ## the health of the `openmetrics_endpoint`. + # + # enable_health_service_check: true + + ## @param ignore_connection_errors - boolean - optional - default: false + ## Whether or not to ignore connection errors when scraping `openmetrics_endpoint`. + # + # ignore_connection_errors: false + + ## @param hostname_label - string - optional + ## Override the hostname for every metric submission with the value of one of its labels. + # + # hostname_label: + + ## @param hostname_format - string - optional + ## When `hostname_label` is set, this instructs the check how to format the values. The string + ## `` is replaced by the value of the label defined by `hostname_label`. + # + # hostname_format: + + ## @param collect_histogram_buckets - boolean - optional - default: true + ## Whether or not to send histogram buckets. + # + # collect_histogram_buckets: true + + ## @param non_cumulative_histogram_buckets - boolean - optional - default: false + ## Whether or not histogram buckets are non-cumulative and to come with a `lower_bound` tag. + # + # non_cumulative_histogram_buckets: false + + ## @param histogram_buckets_as_distributions - boolean - optional - default: false + ## Whether or not to send histogram buckets as Datadog distribution metrics. This implicitly + ## enables the `collect_histogram_buckets` and `non_cumulative_histogram_buckets` options. + ## + ## Learn more about distribution metrics: + ## https://docs.datadoghq.com/developers/metrics/types/?tab=distribution#metric-types + # + # histogram_buckets_as_distributions: false + + ## @param collect_counters_with_distributions - boolean - optional - default: false + ## Whether or not to also collect the observation counter metrics ending in `.sum` and `.count` + ## when sending histogram buckets as Datadog distribution metrics. This implicitly enables the + ## `histogram_buckets_as_distributions` option. + # + # collect_counters_with_distributions: false + + ## @param use_process_start_time - boolean - optional - default: false + ## Whether to enable a heuristic for reporting counter values on the first scrape. When true, + ## the first time an endpoint is scraped, check `process_start_time_seconds` to decide whether zero + ## initial value can be assumed for counters. This requires keeping metrics in memory until the entire + ## response is received. + # + # use_process_start_time: false + + ## @param share_labels - mapping - optional + ## This mapping allows for the sharing of labels across multiple metrics. The keys represent the + ## exposed metrics from which to share labels, and the values are mappings that configure the + ## sharing behavior. Each mapping must have at least one of the following keys: + ## + ## labels - This is a list of labels to share. All labels are shared if this is not set. + ## match - This is a list of labels to match on other metrics as a condition for sharing. + ## values - This is a list of allowed values as a condition for sharing. + ## + ## To unconditionally share all labels of a metric, set it to `true`. + ## + ## For example, the following configuration instructs the check to apply all labels from `metric_a` + ## to all other metrics, the `node` label from `metric_b` to only those metrics that have a `pod` + ## label value that matches the `pod` label value of `metric_b`, and all labels from `metric_c` + ## to all other metrics if their value is equal to `23` or `42`. + ## + ## share_labels: + ## metric_a: true + ## metric_b: + ## labels: + ## - node + ## match: + ## - pod + ## metric_c: + ## values: + ## - 23 + ## - 42 + # + # share_labels: {} + + ## @param cache_shared_labels - boolean - optional - default: true + ## When `share_labels` is set, it instructs the check to cache labels collected from the first payload + ## for improved performance. + ## + ## Set this to `false` to compute label sharing for every payload at the risk of potentially increased memory usage. + # + # cache_shared_labels: true + + ## @param raw_line_filters - list of strings - optional + ## A list of regular expressions used to exclude lines read from the `openmetrics_endpoint` + ## from being parsed. + # + # raw_line_filters: [] + + ## @param cache_metric_wildcards - boolean - optional - default: true + ## Whether or not to cache data from metrics that are defined by regular expressions rather + ## than the full metric name. + # + # cache_metric_wildcards: true + + ## @param telemetry - boolean - optional - default: false + ## Whether or not to submit metrics prefixed by `.telemetry.` for debugging purposes. + # + # telemetry: false + + ## @param ignore_tags - list of strings - optional + ## A list of regular expressions used to ignore tags added by Autodiscovery and entries in the `tags` option. + # + # ignore_tags: + # - + # - + # - + + ## @param proxy - mapping - optional + ## This overrides the `proxy` setting in `init_config`. + ## + ## Set HTTP or HTTPS proxies for this instance. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported, for example: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## This overrides the `skip_proxy` setting in `init_config`. + ## + ## If set to `true`, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + + ## @param auth_type - string - optional - default: basic + ## The type of authentication to use. The available types (and related options) are: + ## + ## - basic + ## |__ username + ## |__ password + ## |__ use_legacy_auth_encoding + ## - digest + ## |__ username + ## |__ password + ## - ntlm + ## |__ ntlm_domain + ## |__ password + ## - kerberos + ## |__ kerberos_auth + ## |__ kerberos_cache + ## |__ kerberos_delegate + ## |__ kerberos_force_initiate + ## |__ kerberos_hostname + ## |__ kerberos_keytab + ## |__ kerberos_principal + ## - aws + ## |__ aws_region + ## |__ aws_host + ## |__ aws_service + ## + ## The `aws` auth type relies on boto3 to automatically gather AWS credentials, for example: from `.aws/credentials`. + ## Details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials + # + # auth_type: basic + + ## @param use_legacy_auth_encoding - boolean - optional - default: true + ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`. + # + # use_legacy_auth_encoding: true + + ## @param username - string - optional + ## The username to use if services are behind basic or digest auth. + # + # username: + + ## @param password - string - optional + ## The password to use if services are behind basic or NTLM auth. + # + # password: + + ## @param ntlm_domain - string - optional + ## If your services use NTLM authentication, specify + ## the domain used in the check. For NTLM Auth, append + ## the username to domain, not as the `username` parameter. + # + # ntlm_domain: \ + + ## @param kerberos_auth - string - optional - default: disabled + ## If your services use Kerberos authentication, you can specify the Kerberos + ## strategy to use between: + ## + ## - required + ## - optional + ## - disabled + ## + ## See https://github.com/requests/requests-kerberos#mutual-authentication + # + # kerberos_auth: disabled + + ## @param kerberos_cache - string - optional + ## Sets the KRB5CCNAME environment variable. + ## It should point to a credential cache with a valid TGT. + # + # kerberos_cache: + + ## @param kerberos_delegate - boolean - optional - default: false + ## Set to `true` to enable Kerberos delegation of credentials to a server that requests delegation. + ## + ## See https://github.com/requests/requests-kerberos#delegation + # + # kerberos_delegate: false + + ## @param kerberos_force_initiate - boolean - optional - default: false + ## Set to `true` to preemptively initiate the Kerberos GSS exchange and + ## present a Kerberos ticket on the initial request (and all subsequent). + ## + ## See https://github.com/requests/requests-kerberos#preemptive-authentication + # + # kerberos_force_initiate: false + + ## @param kerberos_hostname - string - optional + ## Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't + ## match its Kerberos hostname, for example: behind a content switch or load balancer. + ## + ## See https://github.com/requests/requests-kerberos#hostname-override + # + # kerberos_hostname: + + ## @param kerberos_principal - string - optional + ## Set an explicit principal, to force Kerberos to look for a + ## matching credential cache for the named user. + ## + ## See https://github.com/requests/requests-kerberos#explicit-principal + # + # kerberos_principal: + + ## @param kerberos_keytab - string - optional + ## Set the path to your Kerberos key tab file. + # + # kerberos_keytab: + + ## @param auth_token - mapping - optional + ## This allows for the use of authentication information from dynamic sources. + ## Both a reader and writer must be configured. + ## + ## The available readers are: + ## + ## - type: file + ## path (required): The absolute path for the file to read from. + ## pattern: A regular expression pattern with a single capture group used to find the + ## token rather than using the entire file, for example: Your secret is (.+) + ## - type: oauth + ## url (required): The token endpoint. + ## client_id (required): The client identifier. + ## client_secret (required): The client secret. + ## basic_auth: Whether the provider expects credentials to be transmitted in + ## an HTTP Basic Auth header. The default is: false + ## options: Mapping of additional options to pass to the provider, such as the audience + ## or the scope. For example: + ## options: + ## audience: https://example.com + ## scope: read:example + ## + ## The available writers are: + ## + ## - type: header + ## name (required): The name of the field, for example: Authorization + ## value: The template value, for example `Bearer `. The default is: + ## placeholder: The substring in `value` to replace with the token, defaults to: + # + # auth_token: + # reader: + # type: + # : + # : + # writer: + # type: + # : + # : + + ## @param aws_region - string - optional + ## If your services require AWS Signature Version 4 signing, set the region. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_region: + + ## @param aws_host - string - optional + ## If your services require AWS Signature Version 4 signing, set the host. + ## This only needs the hostname and does not require the protocol (HTTP, HTTPS, and more). + ## For example, if connecting to https://us-east-1.amazonaws.com/, set `aws_host` to `us-east-1.amazonaws.com`. + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_host: + + ## @param aws_service - string - optional + ## If your services require AWS Signature Version 4 signing, set the service code. For a list + ## of available service codes, see https://docs.aws.amazon.com/general/latest/gr/rande.html + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_service: + + ## @param tls_verify - boolean - optional - default: true + ## Instructs the check to validate the TLS certificate of services. + # + # tls_verify: true + + ## @param tls_use_host_header - boolean - optional - default: false + ## If a `Host` header is set, this enables its use for SNI (matching against the TLS certificate CN or SAN). + # + # tls_use_host_header: false + + ## @param tls_ignore_warning - boolean - optional - default: false + ## If `tls_verify` is disabled, security warnings are logged by the check. + ## Disable those by setting `tls_ignore_warning` to true. + # + # tls_ignore_warning: false + + ## @param tls_cert - string - optional + ## The path to a single file in PEM format containing a certificate as well as any + ## number of CA certificates needed to establish the certificate's authenticity for + ## use when connecting to services. It may also contain an unencrypted private key to use. + # + # tls_cert: + + ## @param tls_private_key - string - optional + ## The unencrypted private key to use for `tls_cert` when connecting to services. This is + ## required if `tls_cert` is set and it does not already contain a private key. + # + # tls_private_key: + + ## @param tls_ca_cert - string - optional + ## The path to a file of concatenated CA certificates in PEM format or a directory + ## containing several CA certificates in PEM format. If a directory, the directory + ## must have been processed using the `openssl rehash` command. See: + ## https://www.openssl.org/docs/man3.2/man1/c_rehash.html + # + # tls_ca_cert: + + ## @param tls_protocols_allowed - list of strings - optional + ## The expected versions of TLS/SSL when fetching intermediate certificates. + ## Only `SSLv3`, `TLSv1.2`, `TLSv1.3` are allowed by default. The possible values are: + ## SSLv3 + ## TLSv1 + ## TLSv1.1 + ## TLSv1.2 + ## TLSv1.3 + # + # tls_protocols_allowed: + # - SSLv3 + # - TLSv1.2 + # - TLSv1.3 + + ## @param headers - mapping - optional + ## The headers parameter allows you to send specific headers with every request. + ## You can use it for explicitly specifying the host header or adding headers for + ## authorization purposes. + ## + ## This overrides any default headers. + # + # headers: + # Host: + # X-Auth-Token: + + ## @param extra_headers - mapping - optional + ## Additional headers to send with every request. + # + # extra_headers: + # Host: + # X-Auth-Token: + + ## @param timeout - number - optional - default: 10 + ## The timeout for accessing services. + ## + ## This overrides the `timeout` setting in `init_config`. + # + # timeout: 10 + + ## @param connect_timeout - number - optional + ## The connect timeout for accessing services. Defaults to `timeout`. + # + # connect_timeout: + + ## @param read_timeout - number - optional + ## The read timeout for accessing services. Defaults to `timeout`. + # + # read_timeout: + + ## @param request_size - number - optional - default: 16 + ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time. + # + # request_size: 16 + + ## @param log_requests - boolean - optional - default: false + ## Whether or not to debug log the HTTP(S) requests made, including the method and URL. + # + # log_requests: false + + ## @param persist_connections - boolean - optional - default: false + ## Whether or not to persist cookies and use connection pooling for improved performance. + # + # persist_connections: false + + ## @param allow_redirects - boolean - optional - default: true + ## Whether or not to allow URL redirection. + # + # allow_redirects: true + + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 15 + ## This changes the collection interval of the check. For more information, see: + ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + # + # min_collection_interval: 15 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - diff --git a/milvus/datadog_checks/milvus/metrics.py b/milvus/datadog_checks/milvus/metrics.py new file mode 100644 index 0000000000000..bd6a7ff744e59 --- /dev/null +++ b/milvus/datadog_checks/milvus/metrics.py @@ -0,0 +1,231 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +METRIC_MAP = { + 'milvus_build_info': {'type': 'metadata', 'label': 'version', 'name': 'version'}, + 'milvus_cgo_active_future_total': 'cgo.active_future_total', + 'milvus_cgo_cgo_duration_seconds': 'cgo.cgo_duration_seconds', + 'milvus_cgo_cgo_queue_duration_seconds': 'cgo.cgo_queue_duration_seconds', + 'milvus_cgo_running_cgo_call_total': 'cgo.running_cgo_call_total', + 'milvus_datacoord_channel_checkpoint_unix_seconds': { + 'name': 'datacoord.time_since_channel_checkpoint', + 'type': 'time_elapsed', + }, + 'milvus_datacoord_collection_num': 'datacoord.collection_num', + 'milvus_datacoord_consume_datanode_tt_lag_ms': 'datacoord.consume_datanode_tt_lag_ms', + 'milvus_datacoord_datanode_num': 'datacoord.datanode_num', + 'milvus_datacoord_import_tasks': 'datacoord.import_tasks', + 'milvus_datacoord_index_node_num': 'datacoord.index.node_num', + 'milvus_datacoord_index_req_count': 'datacoord.index.req', + 'milvus_datacoord_index_task_count': 'datacoord.index.task', + 'milvus_datacoord_segment_num': 'datacoord.segment_num', + 'milvus_datacoord_stored_binlog_size': 'datacoord.stored.binlog_size', + 'milvus_datacoord_stored_index_files_size': 'datacoord.stored.index_files_size', + 'milvus_datacoord_stored_rows_num': 'datacoord.stored.rows_num', + 'milvus_datacoord_task_execute_max_latency': 'datacoord.task_execute_max_latency', + 'milvus_datacoord_watched_dml_chanel_num': 'datacoord.watched_dml_chanel_num', + 'milvus_datanode_autoflush_buffer_op_count': 'datanode.autoflush_buffer_op', + 'milvus_datanode_consume_bytes_count': 'datanode.consume.bytes', + 'milvus_datanode_consume_msg_count': 'datanode.consume.msg', + 'milvus_datanode_consume_tt_lag_ms': 'datanode.consume.tt_lag_ms', + 'milvus_datanode_encode_buffer_latency': 'datanode.encode_buffer_latency', + 'milvus_datanode_flowgraph_num': 'datanode.flowgraph_num', + 'milvus_datanode_flush_buffer_op_count': 'datanode.flush.buffer_op', + 'milvus_datanode_flush_req_count': 'datanode.flush.req', + 'milvus_datanode_flushed_data_rows': 'datanode.flushed_data.rows', + 'milvus_datanode_flushed_data_size': 'datanode.flushed_data.size', + 'milvus_datanode_msg_dispatcher_tt_lag_ms': 'datanode.msg.dispatcher_tt_lag_ms', + 'milvus_datanode_msg_rows_count': 'datanode.msg.rows', + 'milvus_datanode_save_latency': 'datanode.save_latency', + 'milvus_flushed_segment_file_num': 'flushed_segment_file_num', + 'milvus_indexnode_build_index_latency': 'indexnode.build_index_latency', + 'milvus_indexnode_encode_index_latency': 'indexnode.encode_index_latency', + 'milvus_indexnode_index_task_count': 'indexnode.index.task', + 'milvus_indexnode_index_task_latency_in_queue': 'indexnode.index.task_latency_in_queue', + 'milvus_indexnode_knowhere_build_index_latency': 'indexnode.knowhere_build_index_latency', + 'milvus_indexnode_save_index_latency': 'indexnode.save_index_latency', + 'milvus_meta_kv_size': 'meta.kv_size', + 'milvus_meta_op_count': 'meta.op', + 'milvus_meta_request_latency': 'meta.request_latency', + 'milvus_msg_queue_consumer_num': 'msg_queue_consumer_num', + 'milvus_msgstream_op_count': 'msgstream.op', + 'milvus_msgstream_request_latency': 'msgstream.request_latency', + 'milvus_num_node': 'num_node', + 'milvus_proxy_apply_pk_latency': 'proxy.apply.pk_latency', + 'milvus_proxy_apply_timestamp_latency': 'proxy.apply.timestamp_latency', + 'milvus_proxy_assign_segmentID_latency': 'proxy.assign_segmentID_latency', + 'milvus_proxy_cache_hit_count': 'proxy.cache.hit', + 'milvus_proxy_cache_update_latency': 'proxy.cache.update_latency', + 'milvus_proxy_delete_vectors_count': 'proxy.delete_vectors', + 'milvus_proxy_msgstream_obj_num': 'proxy.msgstream_obj_num', + 'milvus_proxy_mutation_send_latency': 'proxy.mutation_send_latency', + 'milvus_proxy_rate_limit_req_count': 'proxy.rate_limit_req', + 'milvus_proxy_report_value': 'proxy.report_value', + 'milvus_proxy_req_count': 'proxy.req', + 'milvus_proxy_req_in_queue_latency': 'proxy.req.in_queue_latency', + 'milvus_proxy_req_latency': 'proxy.req.latency', + 'milvus_proxy_send_bytes_count': 'proxy.send_bytes', + 'milvus_proxy_sq_decode_result_latency': 'proxy.sq.decode_result_latency', + 'milvus_proxy_sq_reduce_result_latency': 'proxy.sq.reduce_result_latency', + 'milvus_proxy_sq_wait_result_latency': 'proxy.sq.wait_result_latency', + 'milvus_proxy_sync_segment_request_length': 'proxy.sync_segment_request_length', + 'milvus_proxy_tt_lag_ms': 'proxy.tt_lag_ms', + 'milvus_querycoord_collection_num': 'querycoord.collection_num', + 'milvus_querycoord_current_target_checkpoint_unix_seconds': { + 'name': 'querycoord.current_target_checkpoint_unix_seconds', + 'type': 'time_elapsed', + }, + 'milvus_querycoord_load_latency': 'querycoord.load.latency', + 'milvus_querycoord_load_req_count': 'querycoord.load.req', + 'milvus_querycoord_partition_num': 'querycoord.partition_num', + 'milvus_querycoord_querynode_num': 'querycoord.querynode_num', + 'milvus_querycoord_release_latency': 'querycoord.release.latency', + 'milvus_querycoord_release_req_count': 'querycoord.release.req', + 'milvus_querycoord_task_num': 'querycoord_task_num', + 'milvus_querynode_apply_bf_latency': 'querynode.apply_bf_latency', + 'milvus_querynode_collection_num': 'querynode.collection_num', + 'milvus_querynode_consume_bytes_counter': 'querynode.consume.bytes_counter', + 'milvus_querynode_consume_msg_count': 'querynode.consume.msg', + 'milvus_querynode_consume_tt_lag_ms': 'querynode.consume.tt_lag_ms', + 'milvus_querynode_disk_cache_evict_bytes': 'querynode.disk.cache.evict.bytes', + 'milvus_querynode_disk_cache_evict_duration': 'querynode.disk.cache.evict.duration', + 'milvus_querynode_disk_cache_evict_global_duration': 'querynode.disk.cache.evict.global_duration', + 'milvus_querynode_disk_cache_evict': 'querynode.disk.cache.evict', + 'milvus_querynode_disk_cache_load_bytes': 'querynode.disk.cache.load.bytes', + 'milvus_querynode_disk_cache_load_duration': 'querynode.disk.cache.load.duration', + 'milvus_querynode_disk_cache_load_global_duration': 'querynode.disk.cache.load.global_duration', + 'milvus_querynode_disk_cache_load': 'querynode.disk.cache.load', + 'milvus_querynode_disk_used_size': 'querynode.disk.used_size', + 'milvus_querynode_dml_vchannel_num': 'querynode.dml_vchannel_num', + 'milvus_querynode_entity_num': 'querynode.entity.num', + 'milvus_querynode_entity_size': 'querynode.entity.size', + 'milvus_querynode_execute_bytes_counter': 'querynode.execute_bytes_counter', + 'milvus_querynode_flowgraph_num': 'querynode.flowgraph_num', + 'milvus_querynode_forward_delete_latency': 'querynode.forward_delete_latency', + 'milvus_querynode_load_index_latency': 'querynode.load.index_latency', + 'milvus_querynode_load_segment_concurrency': 'querynode.load.segment.concurrency', + 'milvus_querynode_load_segment_latency': 'querynode.load.segment.latency', + 'milvus_querynode_msg_dispatcher_tt_lag_ms': 'querynode.msg_dispatcher_tt_lag_ms', + 'milvus_querynode_partition_num': 'querynode.partition_num', + 'milvus_querynode_process_insert_or_delete_latency': 'querynode.process_insert_or_delete_latency', + 'milvus_querynode_read_task_concurrency': 'querynode.read_task.concurrency', + 'milvus_querynode_read_task_ready_len': 'querynode.read_task.ready_len', + 'milvus_querynode_read_task_unsolved_len': 'querynode.read_task.unsolved_len', + 'milvus_querynode_search_group_nq': 'querynode.search.group.nq', + 'milvus_querynode_search_group_size': 'querynode.search.group.size', + 'milvus_querynode_search_group_topk': 'querynode.search.group.topk', + 'milvus_querynode_search_nq': 'querynode.search.nq', + 'milvus_querynode_search_topk': 'querynode.search.topk', + 'milvus_querynode_segment_access_duration': 'querynode.segment.access.duration', + 'milvus_querynode_segment_access_global_duration': 'querynode.segment.access.global_duration', + 'milvus_querynode_segment_access': 'querynode.segment.access', + 'milvus_querynode_segment_access_wait_cache_duration': 'querynode.segment.access.wait_cache.duration', + 'milvus_querynode_segment_access_wait_cache_global_duration': 'querynode.segment.access.wait_cache.global_duration', + 'milvus_querynode_segment_access_wait_cache': 'querynode.segment.access.wait_cache', + 'milvus_querynode_segment_latency_per_vector': 'querynode.segment.latency_per_vector', + 'milvus_querynode_segment_num': 'querynode.segment.num', + 'milvus_querynode_sq_core_latency': 'querynode.sq.core_latency', + 'milvus_querynode_sq_queue_latency': 'querynode.sq.queue.latency', + 'milvus_querynode_sq_queue_user_latency': 'querynode.sq.queue.user_latency', + 'milvus_querynode_sq_reduce_latency': 'querynode.sq.reduce_latency', + 'milvus_querynode_sq_req_count': 'querynode.sq.req', + 'milvus_querynode_sq_req_latency': 'querynode.sq.req.latency', + 'milvus_querynode_sq_segment_latency': 'querynode.sq.segment_latency', + 'milvus_querynode_sq_wait_tsafe_latency': 'querynode.sq.wait_tsafe_latency', + 'milvus_querynode_wait_processing_msg_count': 'querynode.wait_processing_msg', + 'milvus_querynode_watch_dml_channel_latency': 'querynode.watch_dml_channel_latency', + 'milvus_rootcoord_collection_num': 'rootcoord.collection_num', + 'milvus_rootcoord_credential_num': 'rootcoord.credential_num', + 'milvus_rootcoord_ddl_req_count': 'rootcoord.ddl_req', + 'milvus_rootcoord_ddl_req_latency': 'rootcoord.ddl_req.latency', + 'milvus_rootcoord_ddl_req_latency_in_queue': 'rootcoord.ddl_req.latency_in_queue', + 'milvus_rootcoord_dml_channel_num': 'rootcoord.dml_channel_num', + 'milvus_rootcoord_entity_num': 'rootcoord.entity_num', + 'milvus_rootcoord_force_deny_writing_counter': 'rootcoord.force_deny_writing_counter', + 'milvus_rootcoord_id_alloc_count': 'rootcoord.id_alloc', + 'milvus_rootcoord_indexed_entity_num': 'rootcoord.indexed_entity_num', + 'milvus_rootcoord_msgstream_obj_num': 'rootcoord.msgstream_obj_num', + 'milvus_rootcoord_num_of_roles': 'rootcoord.num_of_roles', + 'milvus_rootcoord_partition_num': 'rootcoord.partition_num', + 'milvus_rootcoord_produce_tt_lag_ms': 'rootcoord.produce_tt_lag_ms', + 'milvus_rootcoord_proxy_num': 'rootcoord.proxy_num', + 'milvus_rootcoord_qn_mem_high_water_level': 'rootcoord.qn_mem_high_water_level', + 'milvus_rootcoord_sync_timetick_latency': 'rootcoord.sync_timetick_latency', + 'milvus_rootcoord_timestamp': 'rootcoord.timestamp', + 'milvus_rootcoord_timestamp_saved': 'rootcoord.timestamp_saved', + 'milvus_runtime_info': 'runtime_info', + 'milvus_storage_kv_size': 'storage.kv_size', + 'milvus_storage_op_count': 'storage.op', + 'milvus_storage_request_latency': 'storage.request_latency', + 'bf_search_cnt': 'bf_search_cnt', + 'bitset_ratio': 'bitset_ratio', + 'build_latency': 'build_latency', + 'cache_hit_cnt': 'cache_hit_cnt', + 'diskann_range_search_iters': 'diskann.range_search_iters', + 'diskann_search_hops': 'diskann.search_hops', + 'diskann_bitset_ratio': 'diskann_bitset_ratio', + 'exec_latency': 'exec_latency', + 'filter_connectivity_ratio': 'filter.connectivity_ratio', + 'filter_mv_activated_fields_cnt': 'filter.mv.activated_fields_cnt', + 'filter_mv_change_base_cnt': 'filter.mv.change_base_cnt', + 'filter_mv_only_cnt': 'filter.mv.only_cnt', + 'filter_mv_supplement_ep_bool_cnt': 'filter.mv.supplement_ep_bool_cnt', + 'go_gc_duration_seconds': 'go.gc_duration_seconds', + 'go_goroutines': 'go.goroutines', + 'go_info': 'go.info', + 'go_memstats_alloc_bytes': {'name': 'go.memstats.alloc_bytes', 'type': 'native_dynamic'}, + 'go_memstats_buck_hash_sys_bytes': 'go.memstats.buck_hash_sys_bytes', + 'go_memstats_frees': 'go.memstats.frees', + 'go_memstats_gc_sys_bytes': 'go.memstats.gc_sys_bytes', + 'go_memstats_heap_alloc_bytes': 'go.memstats.heap.alloc_bytes', + 'go_memstats_heap_idle_bytes': 'go.memstats.heap.idle_bytes', + 'go_memstats_heap_inuse_bytes': 'go.memstats.heap.inuse_bytes', + 'go_memstats_heap_objects': 'go.memstats.heap.objects', + 'go_memstats_heap_released_bytes': 'go.memstats.heap.released_bytes', + 'go_memstats_heap_sys_bytes': 'go.memstats.heap.sys_bytes', + 'go_memstats_last_gc_time_seconds': 'go.memstats.last_gc_time_seconds', + 'go_memstats_lookups': 'go.memstats.lookups', + 'go_memstats_mallocs': 'go.memstats.mallocs', + 'go_memstats_mcache_inuse_bytes': 'go.memstats.mcache.inuse_bytes', + 'go_memstats_mcache_sys_bytes': 'go.memstats.mcache.sys_bytes', + 'go_memstats_mspan_inuse_bytes': 'go.memstats.mspan.inuse_bytes', + 'go_memstats_mspan_sys_bytes': 'go.memstats.mspan.sys_bytes', + 'go_memstats_next_gc_bytes': 'go.memstats.next_gc_bytes', + 'go_memstats_other_sys_bytes': 'go.memstats.other_sys_bytes', + 'go_memstats_stack_inuse_bytes': 'go.memstats.stack.inuse_bytes', + 'go_memstats_stack_sys_bytes': 'go.memstats.stack.sys_bytes', + 'go_memstats_sys_bytes': 'go.memstats.sys_bytes', + 'go_threads': 'go.threads', + 'graph_search_cnt': 'graph_search_cnt', + 'hnsw_bitset_ratio': 'hnsw.bitset_ratio', + 'hnsw_search_hops': 'hnsw.search_hops', + 'internal_core_search_latency': 'internal.core_search_latency', + 'internal_mmap_allocated_space_bytes': 'internal.mmap.allocated_space_bytes', + 'internal_mmap_in_used_space_bytes': 'internal.mmap.in_used_space_bytes', + 'internal_storage_kv_size': 'internal.storage.kv_size', + 'internal_storage_load_duration': 'internal.storage.load_duration', + 'internal_storage_op_count': 'internal.storage.op', + 'internal_storage_request_latency': 'internal.storage.request_latency', + 'io_cnt': 'io_cnt', + 'ivf_search_cnt': 'ivf_search_cnt', + 'load_latency': 'load_latency', + 'process_cpu_seconds': 'process.cpu_seconds', + 'process_max_fds': 'process.max_fds', + 'process_open_fds': 'process.open_fds', + 'process_resident_memory_bytes': 'process.resident_memory_bytes', + 'process_start_time_seconds': {'name': 'process.start_time_seconds', 'type': 'time_elapsed'}, + 'process_virtual_memory_bytes': 'process.virtual_memory.bytes', + 'process_virtual_memory_max_bytes': 'process.virtual_memory.max_bytes', + 'quant_compute_cnt': 'quant.compute_cnt', + 'queue_latency': 'queue.latency', + 'range_search_latency': 'range_search_latency', + 'raw_compute_cnt': 'raw_compute_cnt', + 're_search_cnt': 're_search_cnt', + 'search_latency': 'search.latency', + 'search_topk': 'search.topk', +} + +RENAME_LABELS_MAP = { + 'version': 'milvus_version', +} diff --git a/milvus/hatch.toml b/milvus/hatch.toml new file mode 100644 index 0000000000000..c85c5f07a7df2 --- /dev/null +++ b/milvus/hatch.toml @@ -0,0 +1,4 @@ +[env.collectors.datadog-checks] + +[[envs.default.matrix]] +python = ["3.12"] diff --git a/milvus/manifest.json b/milvus/manifest.json new file mode 100644 index 0000000000000..bf25c410ae353 --- /dev/null +++ b/milvus/manifest.json @@ -0,0 +1,51 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "38ddb395-6770-4b81-9730-e43cf4b4b2a0", + "app_id": "milvus", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Monitor the performance and usage of your Milvus deployments.", + "title": "Milvus", + "media": [], + "classifier_tags": [ + "Supported OS::Linux", + "Supported OS::Windows", + "Supported OS::macOS", + "Category::AI/ML", + "Offering::Integration", + "Submitted Data Type::Metrics", + "Submitted Data Type::Logs" + ] + }, + "assets": { + "integration": { + "auto_install": true, + "source_type_id": 30880529, + "source_type_name": "Milvus", + "configuration": { + "spec": "assets/configuration/spec.yaml" + }, + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "milvus.", + "check": "milvus.build_info", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/milvus/metadata.csv b/milvus/metadata.csv new file mode 100644 index 0000000000000..a845480cf1521 --- /dev/null +++ b/milvus/metadata.csv @@ -0,0 +1,387 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +milvus.bf_search_cnt.bucket,count,,,,Histogram bucket for the number of brute-force searches per request,0,milvus,,, +milvus.bf_search_cnt.count,count,,,,Count aggregation of brute-force searches per request,0,milvus,,, +milvus.bf_search_cnt.sum,count,,,,Sum aggregation of brute-force searches per request,0,milvus,,, +milvus.bitset_ratio.bucket,count,,,,Bitset ratio histogram bucket,0,milvus,,, +milvus.bitset_ratio.count,count,,,,Count of bitset ratio calculations,0,milvus,,, +milvus.bitset_ratio.sum,count,,,,Sum of bitset ratios,0,milvus,,, +milvus.build_info,gauge,,,,Build information of milvus,0,milvus,,, +milvus.build_latency.bucket,count,,,,Index build latency histogram bucket,0,milvus,,, +milvus.build_latency.count,count,,,,Count of index builds,0,milvus,,, +milvus.build_latency.sum,count,,second,,Sum of index build latencies(s),0,milvus,,, +milvus.cache_hit_cnt.bucket,count,,,,Histogram bucket for the cache hit count per request,0,milvus,,, +milvus.cache_hit_cnt.count,count,,,,Count aggregation of cache hit count per request,0,milvus,,, +milvus.cache_hit_cnt.sum,count,,,,Sum aggregation of cache hit count per request,0,milvus,,, +milvus.cgo.active_future_total,gauge,,,,Total number of active futures.,0,milvus,,, +milvus.cgo.cgo_duration_seconds.bucket,count,,,,Histogram bucket of cgo call duration in seconds,0,milvus,,, +milvus.cgo.cgo_duration_seconds.count,count,,,,Count of cgo calls,0,milvus,,, +milvus.cgo.cgo_duration_seconds.sum,count,,second,,Sum of cgo call duration in seconds.,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.bucket,count,,,,Histogram bucket for the duration of cgo calls in queue,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.count,count,,,,Counnt of cgo calls in queue,0,milvus,,, +milvus.cgo.cgo_queue_duration_seconds.sum,count,,second,,Sum of the duration of cgo calls in queue,0,milvus,,, +milvus.cgo.running_cgo_call_total,gauge,,,,Total number of running cgo calls.,0,milvus,,, +milvus.datacoord.channel_checkpoint_unix_seconds,gauge,,,,Channel checkpoint timestamp in Unix seconds,0,milvus,,, +milvus.datacoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.datacoord.consume_datanode_tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, +milvus.datacoord.datanode_num,gauge,,,,Number of data nodes,0,milvus,,, +milvus.datacoord.import_tasks,gauge,,,,The import tasks grouping by type and state,0,milvus,,, +milvus.datacoord.index.node_num,gauge,,,,Number of IndexNodes managed by IndexCoord,0,milvus,,, +milvus.datacoord.index.req.count,count,,,,Number of building index requests,0,milvus,,, +milvus.datacoord.index.task,gauge,,,,Number of index tasks of each type,0,milvus,,, +milvus.datacoord.segment_num,gauge,,,,Number of segments,0,milvus,,, +milvus.datacoord.stored.binlog_size,gauge,,byte,,Binlog size of healthy segments,0,milvus,,, +milvus.datacoord.stored.index_files_size,gauge,,byte,,Index files size of the segments,0,milvus,,, +milvus.datacoord.stored.rows_num,gauge,,,,Number of stored rows of healthy segment,0,milvus,,, +milvus.datacoord.task_execute_max_latency.bucket,count,,,,Histogram bucket for the latency of task execute operation,0,milvus,,, +milvus.datacoord.task_execute_max_latency.count,count,,,,Count of task execute operations,0,milvus,,, +milvus.datacoord.task_execute_max_latency.sum,count,,millisecond,,Sum of the latency of task execute operations,0,milvus,,, +milvus.datacoord.watched_dml_chanel_num,gauge,,,,The num of data manipulation language channel watched by datanode,0,milvus,,, +milvus.datanode.autoflush_buffer_op.count,count,,,,Count of auto flush buffer operations,0,milvus,,, +milvus.datanode.consume.bytes.count,count,,,,Count of consumed bytes,0,milvus,,, +milvus.datanode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, +milvus.datanode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, +milvus.datanode.encode_buffer_latency.bucket,count,,,,Histogram bucket for the latency of encode buffer data,0,milvus,,, +milvus.datanode.encode_buffer_latency.count,count,,,,Count of encode buffer data operations,0,milvus,,, +milvus.datanode.encode_buffer_latency.sum,count,,millisecond,,Sum of the latency of encode buffer data operations,0,milvus,,, +milvus.datanode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, +milvus.datanode.flush.buffer_op.count,count,,,,Count of flush buffer operations,0,milvus,,, +milvus.datanode.flush.req.count,count,,,,Count of flush request,0,milvus,,, +milvus.datanode.flushed_data.rows.count,count,,,,Num of rows flushed to storage,0,milvus,,, +milvus.datanode.flushed_data.size.count,count,,byte,,Byte size of data flushed to storage,0,milvus,,, +milvus.datanode.msg.dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, +milvus.datanode.msg.rows.count,count,,,,Count of rows consumed from msgStream,0,milvus,,, +milvus.datanode.save_latency.bucket,count,,,,Histogram bucket for the latency of saving flush data to storage,0,milvus,,, +milvus.datanode.save_latency.count,count,,,,Count of saving flush data to storage events,0,milvus,,, +milvus.datanode.save_latency.sum,count,,millisecond,,Sum of the latency of saving flush data to storage events,0,milvus,,, +milvus.diskann.range_search_iters.bucket,count,,,,Histogram bucket for DISKANN range search iterations,0,milvus,,, +milvus.diskann.range_search_iters.count,count,,,,Count of DISKANN range search events,0,milvus,,, +milvus.diskann.range_search_iters.sum,count,,,,Sum of DISKANN range search iterations,0,milvus,,, +milvus.diskann.search_hops.bucket,count,,,,Histogram for DISKANN search hops,0,milvus,,, +milvus.diskann.search_hops.count,count,,,,Count of DISKANN search hop events,0,milvus,,, +milvus.diskann.search_hops.sum,count,,,,Sum of DISKANN search hops,0,milvus,,, +milvus.diskann_bitset_ratio.bucket,count,,,,Histogram bucket for the DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.count,count,,,,Count of the DISKANN bitset ratio operations for search and range search,0,milvus,,, +milvus.diskann_bitset_ratio.sum,count,,,,Sum of the DISKANN bitset ratio for search and range search,0,milvus,,, +milvus.exec_latency.bucket,count,,,,Histogram bucket for the execute latency per request,0,milvus,,, +milvus.exec_latency.count,count,,,,Count aggregation of the execute latency per request,0,milvus,,, +milvus.exec_latency.sum,count,,,,Sum aggregation of the execute latency per request,0,milvus,,, +milvus.filter.connectivity_ratio.bucket,count,,,,Histogram bucket for the average connectivity ratio set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.count,count,,,,Count of the connectivity ratios set under filtering per request,0,milvus,,, +milvus.filter.connectivity_ratio.sum,count,,,,Sum of the average connectivity ratios set under filtering per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.bucket,count,,,,Histogram bucket of the average materialized view activated fields per request,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.count,count,,,,Count of the requests activating materialized view fields,0,milvus,,, +milvus.filter.mv.activated_fields_cnt.sum,count,,,,Sum of the average materialized view activated fields per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.bucket,count,,,,Histogram bucket for the materialized view change base count per request,0,milvus,,, +milvus.filter.mv.change_base_cnt.count,count,,,,Count of requests triggering materialized view change base events,0,milvus,,, +milvus.filter.mv.change_base_cnt.sum,count,,,,Sum of materialized view change base counts per request,0,milvus,,, +milvus.filter.mv.only_cnt.bucket,count,,,,Histogram bucket for the materialized view only count per request,0,milvus,,, +milvus.filter.mv.only_cnt.count,count,,,,Count of requests that trigger materialized view only events,0,milvus,,, +milvus.filter.mv.only_cnt.sum,count,,,,Sum of materialized view only counts,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.bucket,count,,,,Histogram bucket for materialized view supplement entry point from bitset boolean count per request,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.count,count,,,,Count of requests triggering a materialized view supplement entry point from bitset boolean event,0,milvus,,, +milvus.filter.mv.supplement_ep_bool_cnt.sum,count,,,,Sum of materialized view supplement entry point from bitset boolean events,0,milvus,,, +milvus.flushed_segment_file_num.bucket,count,,,,Histogram bucket for the number of files for flushed segment,0,milvus,,, +milvus.flushed_segment_file_num.count,count,,,,Count of flushed segment events,0,milvus,,, +milvus.flushed_segment_file_num.sum,count,,,,Sum of the number of files for flushed segments,0,milvus,,, +milvus.go.gc_duration_seconds.count,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.gc_duration_seconds.quantile,gauge,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.gc_duration_seconds.sum,count,,,,A summary of the pause duration of garbage collection cycles.,0,milvus,,, +milvus.go.goroutines,gauge,,,,Number of goroutines that currently exist.,0,milvus,,, +milvus.go.info,gauge,,,,Information about the Go environment.,0,milvus,,, +milvus.go.memstats.alloc_bytes,gauge,,,,Number of bytes allocated and still in use.,0,milvus,,, +milvus.go.memstats.alloc_bytes.count,count,,,,"Total number of bytes allocated, even if freed.",0,milvus,,, +milvus.go.memstats.buck_hash_sys_bytes,gauge,,,,Number of bytes used by the profiling bucket hash table.,0,milvus,,, +milvus.go.memstats.frees.count,count,,,,Total number of frees.,0,milvus,,, +milvus.go.memstats.gc_sys_bytes,gauge,,,,Number of bytes used for garbage collection system metadata.,0,milvus,,, +milvus.go.memstats.heap.alloc_bytes,gauge,,,,Number of heap bytes allocated and still in use.,0,milvus,,, +milvus.go.memstats.heap.idle_bytes,gauge,,,,Number of heap bytes waiting to be used.,0,milvus,,, +milvus.go.memstats.heap.inuse_bytes,gauge,,,,Number of heap bytes that are in use.,0,milvus,,, +milvus.go.memstats.heap.objects,gauge,,,,Number of allocated objects.,0,milvus,,, +milvus.go.memstats.heap.released_bytes,gauge,,,,Number of heap bytes released to OS.,0,milvus,,, +milvus.go.memstats.heap.sys_bytes,gauge,,,,Number of heap bytes obtained from system.,0,milvus,,, +milvus.go.memstats.last_gc_time_seconds,gauge,,,,Number of seconds since 1970 of last garbage collection.,0,milvus,,, +milvus.go.memstats.lookups.count,count,,,,Total number of pointer lookups.,0,milvus,,, +milvus.go.memstats.mallocs.count,count,,,,Total number of mallocs.,0,milvus,,, +milvus.go.memstats.mcache.inuse_bytes,gauge,,,,Number of bytes in use by mcache structures.,0,milvus,,, +milvus.go.memstats.mcache.sys_bytes,gauge,,,,Number of bytes used for mcache structures obtained from system.,0,milvus,,, +milvus.go.memstats.mspan.inuse_bytes,gauge,,,,Number of bytes in use by mspan structures.,0,milvus,,, +milvus.go.memstats.mspan.sys_bytes,gauge,,,,Number of bytes used for mspan structures obtained from system.,0,milvus,,, +milvus.go.memstats.next_gc_bytes,gauge,,,,Number of heap bytes when next garbage collection will take place.,0,milvus,,, +milvus.go.memstats.other_sys_bytes,gauge,,,,Number of bytes used for other system allocations.,0,milvus,,, +milvus.go.memstats.stack.inuse_bytes,gauge,,,,Number of bytes in use by the stack allocator.,0,milvus,,, +milvus.go.memstats.stack.sys_bytes,gauge,,,,Number of bytes obtained from system for stack allocator.,0,milvus,,, +milvus.go.memstats.sys_bytes,gauge,,,,Number of bytes obtained from system.,0,milvus,,, +milvus.go.threads,gauge,,,,Number of OS threads created.,0,milvus,,, +milvus.graph_search_cnt.bucket,count,,,,Histogram bucket for the number of graph search per request,0,milvus,,, +milvus.graph_search_cnt.count,count,,,,Count of requests that trigger a graph search,0,milvus,,, +milvus.graph_search_cnt.sum,count,,,,Sum of graph searches,0,milvus,,, +milvus.hnsw.bitset_ratio.bucket,count,,,,Histogram bucket of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.count,count,,,,Count aggregation of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.bitset_ratio.sum,count,,,,Sum aggregation of the HNSW bitset ratio for search and range search,0,milvus,,, +milvus.hnsw.search_hops.bucket,count,,,,Histogram bucket of the HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.count,count,,,,Count aggregation of the HNSW search hops in layer 0,0,milvus,,, +milvus.hnsw.search_hops.sum,count,,,,Sum aggregation of the HNSW search hops in layer 0,0,milvus,,, +milvus.indexnode.build_index_latency.bucket,count,,,,Histogram bucket of the latency of build index for segment,0,milvus,,, +milvus.indexnode.build_index_latency.count,count,,,,Count of build index events,0,milvus,,, +milvus.indexnode.build_index_latency.sum,count,,millisecond,,Sum of the latency of build index events,0,milvus,,, +milvus.indexnode.encode_index_latency.bucket,count,,,,Histogram bucket of the latency of encoding the index file,0,milvus,,, +milvus.indexnode.encode_index_latency.count,count,,,,Count of the index file encoding events,0,milvus,,, +milvus.indexnode.encode_index_latency.sum,count,,millisecond,,Sum of the latency of index file encoding events,0,milvus,,, +milvus.indexnode.index.task.count,count,,,,Number of tasks that index node received,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.bucket,count,,,,Histogram bucket for the latency of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.count,count,,,,Count of index task in queue,0,milvus,,, +milvus.indexnode.index.task_latency_in_queue.sum,count,,millisecond,,Sum of the latency of index task in queue,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.bucket,count,,,,Histogram bucket for the latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.count,count,,,,Count of index builds by knowhere,0,milvus,,, +milvus.indexnode.knowhere_build_index_latency.sum,count,,millisecond,,Sum of the latency of building the index by knowhere,0,milvus,,, +milvus.indexnode.save_index_latency.bucket,count,,,,Histogram bucket for the latency of saving the index file,0,milvus,,, +milvus.indexnode.save_index_latency.count,count,,,,Count of index file save envents,0,milvus,,, +milvus.indexnode.save_index_latency.sum,count,,millisecond,,Sum of the latency of saving the index file,0,milvus,,, +milvus.internal.core_search_latency.bucket,count,,,,Histogram bucket for the [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.count,count,,,,Count aggregation of [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.core_search_latency.sum,count,,microsecond,,Sum aggregation of [cpp]latency(us) of search on segment,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.bucket,count,,,,Histogram bucket for the [cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.count,count,,,,Count aggregation of [cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.allocated_space_bytes.sum,count,,,,Sum aggregation of [cpp]mmap allocated space stats,0,milvus,,, +milvus.internal.mmap.in_used_space_bytes,gauge,,,,[cpp]mmap in used space stats,0,milvus,,, +milvus.internal.storage.kv_size.bucket,count,,,,Histogram bucket for the [cpp]key-value size stats,0,milvus,,, +milvus.internal.storage.kv_size.count,count,,,,Count aggregation of [cpp]key-value size stats,0,milvus,,, +milvus.internal.storage.kv_size.sum,count,,,,Sum aggregation of [cpp]key-value size stats,0,milvus,,, +milvus.internal.storage.load_duration.bucket,count,,,,Histogram bucket for the [cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.count,count,,,,Count aggregation of [cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.load_duration.sum,count,,,,Sum aggregation of [cpp]durations of load segment,0,milvus,,, +milvus.internal.storage.op.count,count,,,,[cpp]count of persistent data operation,0,milvus,,, +milvus.internal.storage.request_latency.bucket,count,,,,Histogram bucket for the [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.count,count,,,,Count aggregation of [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.internal.storage.request_latency.sum,count,,millisecond,,Sum aggregation of [cpp]request latency(ms) on the client side,0,milvus,,, +milvus.io_cnt.bucket,count,,,,Histogram bucket for the IO count per request,0,milvus,,, +milvus.io_cnt.count,count,,,,Count of requests triggering IO operations,0,milvus,,, +milvus.io_cnt.sum,count,,,,Sum of IO operation counts,0,milvus,,, +milvus.ivf_search_cnt.bucket,count,,,,Histogram bucket for the number of inverted file search per request,0,milvus,,, +milvus.ivf_search_cnt.count,count,,,,Count of requests trigerring an inverted file search,0,milvus,,, +milvus.ivf_search_cnt.sum,count,,,,Sum of inverted file searches,0,milvus,,, +milvus.load_latency.bucket,count,,,,Histogram bucket for the index load latency (ms),0,milvus,,, +milvus.load_latency.count,count,,,,Count of index load events,0,milvus,,, +milvus.load_latency.sum,count,,millisecond,,Sum of index load latency (ms),0,milvus,,, +milvus.meta.kv_size.bucket,count,,,,Histogram for the key-value size stats,0,milvus,,, +milvus.meta.kv_size.count,count,,,,Count aggregation of the key-value size stats,0,milvus,,, +milvus.meta.kv_size.sum,count,,,,Sum aggregation of the key-value size stats,0,milvus,,, +milvus.meta.op.count,count,,,,Count of meta operation,0,milvus,,, +milvus.meta.request_latency.bucket,count,,,,Histogram bucket for the request latency on the client side,0,milvus,,, +milvus.meta.request_latency.count,count,,,,Count aggregation of the request latency on the client side,0,milvus,,, +milvus.meta.request_latency.sum,count,,millisecond,,Sum aggregation of the request latency on the client side,0,milvus,,, +milvus.msg_queue_consumer_num,gauge,,,,Number of consumers,0,milvus,,, +milvus.msgstream.op.count,count,,,,Count of stream message operation,0,milvus,,, +milvus.msgstream.request_latency.bucket,count,,,,Histogram bucket for the request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.count,count,,,,Count aggregation of the request latency on the client side,0,milvus,,, +milvus.msgstream.request_latency.sum,count,,millisecond,,Sum aggregation of the request latency on the client side,0,milvus,,, +milvus.num_node,gauge,,,,Number of nodes and coordinates,0,milvus,,, +milvus.process.cpu_seconds.count,count,,,,Total user and system CPU time spent in seconds.,0,milvus,,, +milvus.process.max_fds,gauge,,,,Maximum number of open file descriptors.,0,milvus,,, +milvus.process.open_fds,gauge,,,,Number of open file descriptors.,0,milvus,,, +milvus.process.resident_memory_bytes,gauge,,,,Resident memory size in bytes.,0,milvus,,, +milvus.process.start_time_seconds,gauge,,,,Start time of the process since Unix epoch in seconds.,0,milvus,,, +milvus.process.virtual_memory.bytes,gauge,,,,Virtual memory size in bytes.,0,milvus,,, +milvus.process.virtual_memory.max_bytes,gauge,,,,Maximum amount of virtual memory available in bytes.,0,milvus,,, +milvus.proxy.apply.pk_latency.bucket,count,,,,"Histogram bucket for the latency of 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.pk_latency.count,count,,,,"Count of 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.pk_latency.sum,count,,millisecond,,"Sum of latencies for 'apply primary key' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.count,count,,,,"Count of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.apply.timestamp_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'apply timestamp' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'get segmentID from dataCoord' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.count,count,,,,"Count of proxy 'get segmentID from dataCoord' events",0,milvus,,, +milvus.proxy.assign_segmentID_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'get segmentID from dataCoord' events",0,milvus,,, +milvus.proxy.cache.hit.count,count,,,,Count of cache hits/miss,0,milvus,,, +milvus.proxy.cache.update_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'update cache when cache miss' events",0,milvus,,, +milvus.proxy.cache.update_latency.count,count,,,,"Count of proxy 'update cache when cache miss' events",0,milvus,,, +milvus.proxy.cache.update_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'update cache when cache miss'",0,milvus,,, +milvus.proxy.delete_vectors.count,count,,,,Counter of vectors successfully deleted,0,milvus,,, +milvus.proxy.msgstream_obj_num,gauge,,,,Number of MsgStream objects per physical channel,0,milvus,,, +milvus.proxy.mutation_send_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'send insert request to MsgStream' events",0,milvus,,, +milvus.proxy.mutation_send_latency.count,count,,,,"Count of proxy 'send insert request to MsgStream' events",0,milvus,,, +milvus.proxy.mutation_send_latency.sum,count,,millisecond,,"Sum of latencies of proxy 'send insert request to MsgStream' events",0,milvus,,, +milvus.proxy.rate_limit_req.count,count,,,,Count of operation executed,0,milvus,,, +milvus.proxy.report_value.count,count,,,,Report value about the request,0,milvus,,, +milvus.proxy.req.count,count,,,,Count of operation executed,0,milvus,,, +milvus.proxy.req.in_queue_latency.bucket,count,,,,Histogram bucket for the time in queue of request,0,milvus,,, +milvus.proxy.req.in_queue_latency.count,count,,,,Count of request in queue,0,milvus,,, +milvus.proxy.req.in_queue_latency.sum,count,,millisecond,,Sum of the time requests spend in queue,0,milvus,,, +milvus.proxy.req.latency.bucket,count,,,,Histogram bucket for the latency of each request,0,milvus,,, +milvus.proxy.req.latency.count,count,,,,Count of request latency events,0,milvus,,, +milvus.proxy.req.latency.sum,count,,millisecond,,Sum of latencies of requests,0,milvus,,, +milvus.proxy.send_bytes.count,count,,,,Count of bytes sent back to sdk,0,milvus,,, +milvus.proxy.sq.decode_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.decode_result_latency.count,count,,,,"Count of proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.decode_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'decode the search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.count,count,,,,"Count of proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.reduce_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'reduces search result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.bucket,count,,,,"Histogram bucket for the latency of proxy 'waits for the result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.count,count,,,,"Count of proxy 'waits for the result' events",0,milvus,,, +milvus.proxy.sq.wait_result_latency.sum,count,,millisecond,,"Sum of latencies for proxy 'waits for the result' events",0,milvus,,, +milvus.proxy.sync_segment_request_length.bucket,count,,,,Histogram bucket for the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.sync_segment_request_length.count,count,,,,Count of assigning segments for insert events,0,milvus,,, +milvus.proxy.sync_segment_request_length.sum,count,,,,Sum of the length of SegmentIDRequests when assigning segments for insert,0,milvus,,, +milvus.proxy.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, +milvus.quant.compute_cnt.bucket,count,,,,Histogram bucket for the quant compute count per request,0,milvus,,, +milvus.quant.compute_cnt.count,count,,,,Count of requests triggering quant compute events,0,milvus,,, +milvus.quant.compute_cnt.sum,count,,,,Sum of quant compute event counts,0,milvus,,, +milvus.querycoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.querycoord.current_target_checkpoint_unix_seconds,gauge,,,,Current target checkpoint timestamp in Unix seconds,0,milvus,,, +milvus.querycoord.load.latency.bucket,count,,,,Histogram bucket Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.count,count,,,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.latency.sum,count,,millisecond,,Latency of load the entire collection,0,milvus,,, +milvus.querycoord.load.req.count,count,,,,Count of load request,0,milvus,,, +milvus.querycoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, +milvus.querycoord.querynode_num,gauge,,,,Number of QueryNodes managered by QueryCoord,0,milvus,,, +milvus.querycoord.release.latency.bucket,count,,,,Histogram bucket for the latency of release request,0,milvus,,, +milvus.querycoord.release.latency.count,count,,,,Count of release request events,0,milvus,,, +milvus.querycoord.release.latency.sum,count,,millisecond,,Sum of the latencies of release request events,0,milvus,,, +milvus.querycoord.release.req.count,count,,,,Count of release request,0,milvus,,, +milvus.querycoord_task_num,gauge,,,,the number of tasks in QueryCoord's scheduler,0,milvus,,, +milvus.querynode.apply_bf_latency.bucket,count,,,,Histogram bucket for the apply brute-force cost in ms,0,milvus,,, +milvus.querynode.apply_bf_latency.count,count,,,,Count of apply brute-force events,0,milvus,,, +milvus.querynode.apply_bf_latency.sum,count,,millisecond,,Sum of the apply brute-force cost in ms,0,milvus,,, +milvus.querynode.collection_num,gauge,,,,Number of collections loaded,0,milvus,,, +milvus.querynode.consume.bytes_counter.count,count,,,,Count of consumed bytes,0,milvus,,, +milvus.querynode.consume.msg.count,count,,,,Count of consumed msg,0,milvus,,, +milvus.querynode.consume.tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, +milvus.querynode.disk.cache.evict.bytes.count,count,,byte,,Number of bytes evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.count,count,,,,Number of segments evicted from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.duration.count,count,,,,Total time cost of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.bucket,count,,,,Histogram bucket for the global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.count,count,,,,Count of evicting segments from disk cache events,0,milvus,,, +milvus.querynode.disk.cache.evict.global_duration.sum,count,,millisecond,,Sum of the global duration of evicting segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.bytes.count,count,,byte,,Number of bytes loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.count,count,,,,Number of segments loaded from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.duration.count,count,,,,Total time cost of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.bucket,count,,,,Histogram bucket for the global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.count,count,,,,Count of loading segments from disk cache events,0,milvus,,, +milvus.querynode.disk.cache.load.global_duration.sum,count,,millisecond,,Sum of the global duration of loading segments from disk cache,0,milvus,,, +milvus.querynode.disk.used_size,gauge,,mebibyte,,Disk used size(MB),0,milvus,,, +milvus.querynode.dml_vchannel_num,gauge,,,,Number of data manipulation language channels watched,0,milvus,,, +milvus.querynode.entity.num,gauge,,,,"Number of entities which can be searched/queried, clustered by collection, partition and state",0,milvus,,, +milvus.querynode.entity.size,gauge,,byte,,"Entities' memory size, clustered by collection and state",0,milvus,,, +milvus.querynode.execute_bytes_counter.count,count,,,,Count of execute bytes,0,milvus,,, +milvus.querynode.flowgraph_num,gauge,,,,Number of flowgraphs,0,milvus,,, +milvus.querynode.forward_delete_latency.bucket,count,,,,Histogram bucket for the forward delete cost in ms,0,milvus,,, +milvus.querynode.forward_delete_latency.count,count,,,,Count of forward delete events,0,milvus,,, +milvus.querynode.forward_delete_latency.sum,count,,millisecond,,Sum of the forward delete cost in ms,0,milvus,,, +milvus.querynode.load.index_latency.bucket,count,,,,"Histogram bucket for the latency of load per segment's index, in milliseconds",0,milvus,,, +milvus.querynode.load.index_latency.count,count,,,,"Count of load per segment's index events",0,milvus,,, +milvus.querynode.load.index_latency.sum,count,,millisecond,,"Sum of the latencies of load per segment's index events, in milliseconds",0,milvus,,, +milvus.querynode.load.segment.concurrency,gauge,,,,Number of concurrent loading segments in QueryNode,0,milvus,,, +milvus.querynode.load.segment.latency.bucket,count,,,,Histogram bucket for the latency of load per segment,0,milvus,,, +milvus.querynode.load.segment.latency.count,count,,,,Count of load per segment events,0,milvus,,, +milvus.querynode.load.segment.latency.sum,count,,millisecond,,Sum of the latencies of load per segment events,0,milvus,,, +milvus.querynode.msg_dispatcher_tt_lag_ms,gauge,,millisecond,,time.Now() sub dispatcher's current consume time,0,milvus,,, +milvus.querynode.partition_num,gauge,,,,Number of partitions loaded,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.bucket,count,,,,Histogram bucket for the process insert or delete cost in ms,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.count,count,,,,Count of process insert or delete events,0,milvus,,, +milvus.querynode.process_insert_or_delete_latency.sum,count,,millisecond,,Sum of process insert or delete cost in ms,0,milvus,,, +milvus.querynode.read_task.concurrency,gauge,,,,Number of concurrent executing read tasks in QueryNode,0,milvus,,, +milvus.querynode.read_task.ready_len,gauge,,,,Number of ready read tasks in readyQueue,0,milvus,,, +milvus.querynode.read_task.unsolved_len,gauge,,,,Number of unsolved read tasks in unsolvedQueue,0,milvus,,, +milvus.querynode.search.group.nq.bucket,count,,,,Histogram bucket for the number of queries of each grouped search task,0,milvus,,, +milvus.querynode.search.group.nq.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.nq.sum,count,,,,Sum of the number of queries of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.size.bucket,count,,,,Histogram bucket for the number of tasks of each grouped search task,0,milvus,,, +milvus.querynode.search.group.size.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.size.sum,count,,,,Sum of the number of tasks of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.topk.bucket,count,,,,Histogram bucket for the topK of each grouped search task,0,milvus,,, +milvus.querynode.search.group.topk.count,count,,,,Count of grouped search tasks,0,milvus,,, +milvus.querynode.search.group.topk.sum,count,,,,Sum of the topK of grouped search tasks,0,milvus,,, +milvus.querynode.search.nq.bucket,count,,,,Histogram bucket for the number of queries of each search task,0,milvus,,, +milvus.querynode.search.nq.count,count,,,,Count of search tasks,0,milvus,,, +milvus.querynode.search.nq.sum,count,,,,Sum of the number of queries of search tasks,0,milvus,,, +milvus.querynode.search.topk.bucket,count,,,,Histogram bucket for the top of each search task,0,milvus,,, +milvus.querynode.search.topk.count,count,,,,Count of search tasks,0,milvus,,, +milvus.querynode.search.topk.sum,count,,,,Sum of the top of search tasks,0,milvus,,, +milvus.querynode.segment.access.count,count,,,,Number of segments accessed,0,milvus,,, +milvus.querynode.segment.access.duration.count,count,,,,Total time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.bucket,count,,,,Histogram bucket for the global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.global_duration.count,count,,,,Count of accessing segments events,0,milvus,,, +milvus.querynode.segment.access.global_duration.sum,count,,millisecond,,Sum of the global time cost of accessing segments,0,milvus,,, +milvus.querynode.segment.access.wait_cache.count,count,,,,Number of segments waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.duration.count,count,,,,Total time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.bucket,count,,,,Histogram bucket for the global time cost of waiting for loading access,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.count,count,,,,Count of waiting for loading access events,0,milvus,,, +milvus.querynode.segment.access.wait_cache.global_duration.sum,count,,millisecond,,Sum of the global time cost of waiting for loading access events,0,milvus,,, +milvus.querynode.segment.latency_per_vector.bucket,count,,,,Histogram bucket for one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.count,count,,,,Count aggregation of one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.latency_per_vector.sum,count,,millisecond,,Sum aggregation of one vector's search latency per segment,0,milvus,,, +milvus.querynode.segment.num,gauge,,,,"Number of segments loaded, clustered by its collection, partition, state and # of indexed fields",0,milvus,,, +milvus.querynode.sq.core_latency.bucket,count,,,,Histogram bucket for the latency of searches or queries in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.count,count,,,,Count of search or query events in segcore,0,milvus,,, +milvus.querynode.sq.core_latency.sum,count,,millisecond,,Sum of latencies of searches or queries in segcore,0,milvus,,, +milvus.querynode.sq.queue.latency.bucket,count,,,,Histogram bucket for the latency of searches or queries in queue,0,milvus,,, +milvus.querynode.sq.queue.latency.count,count,,,,Count of search or query events,0,milvus,,, +milvus.querynode.sq.queue.latency.sum,count,,millisecond,,Sum of latencies of searches or queries in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.bucket,count,,,,Histgram bucket for the latency per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.count,count,,,,Count of search or query in queue,0,milvus,,, +milvus.querynode.sq.queue.user_latency.sum,count,,millisecond,,Sum of latencies per user of search or query in queue,0,milvus,,, +milvus.querynode.sq.reduce_latency.bucket,count,,,,Histogram bucket for the latencies of reduce search or query result,0,milvus,,, +milvus.querynode.sq.reduce_latency.count,count,,,,Count of reduce search or query result events,0,milvus,,, +milvus.querynode.sq.reduce_latency.sum,count,,millisecond,,Sum of latencies of reduce search or query result,0,milvus,,, +milvus.querynode.sq.req.count,count,,,,count of search / query request,0,milvus,,, +milvus.querynode.sq.req.latency.bucket,count,,,,Histogram bucket for the latency of search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.count,count,,,,Count of search or query requests,0,milvus,,, +milvus.querynode.sq.req.latency.sum,count,,millisecond,,Sum of latencies of search or query requests,0,milvus,,, +milvus.querynode.sq.segment_latency.bucket,count,,,,Histogram bucket for the latency of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.count,count,,,,Count of search or query per segment,0,milvus,,, +milvus.querynode.sq.segment_latency.sum,count,,millisecond,,Sum of latencies of search or query per segment,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.bucket,count,,,,Histogram bucket for the time that search or query waits for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.count,count,,,,Count of occurrences of search or query waiting for tsafe,0,milvus,,, +milvus.querynode.sq.wait_tsafe_latency.sum,count,,millisecond,,Sum of the time that search or query wait for tsafe,0,milvus,,, +milvus.querynode.wait_processing_msg,gauge,,,,Count of wait processing msg,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.bucket,count,,,,Histogram bucket for the latencies of watch data manipulation language channel,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.count,count,,,,Count of watch data manipulation language channel events,0,milvus,,, +milvus.querynode.watch_dml_channel_latency.sum,count,,millisecond,,Sum of the latencies of watch data manipulation language channel events,0,milvus,,, +milvus.queue.latency.bucket,count,,,,Histogram bucket for the queue latency per request,0,milvus,,, +milvus.queue.latency.count,count,,,,Count of request with queue latency,0,milvus,,, +milvus.queue.latency.sum,count,,,,Sum of queue latencies,0,milvus,,, +milvus.range_search_latency.bucket,count,,,,Histogram bucket for range search latency (ms),0,milvus,,, +milvus.range_search_latency.count,count,,,,Count of range search operations,0,milvus,,, +milvus.range_search_latency.sum,count,,millisecond,,Sum of range search latencies (ms),0,milvus,,, +milvus.raw_compute_cnt.bucket,count,,,,Histogram bucket for the raw compute count per request,0,milvus,,, +milvus.raw_compute_cnt.count,count,,,,Count of request that trigger raw compute operations,0,milvus,,, +milvus.raw_compute_cnt.sum,count,,,,Sum of raw compute operations,0,milvus,,, +milvus.re_search_cnt.bucket,count,,,,Histogram bucket for the number of fallback search per request,0,milvus,,, +milvus.re_search_cnt.count,count,,,,Count of requests triggering fallback search operations,0,milvus,,, +milvus.re_search_cnt.sum,count,,,,Sum of fallback searches,0,milvus,,, +milvus.rootcoord.collection_num,gauge,,,,Number of collections,0,milvus,,, +milvus.rootcoord.credential_num,gauge,,,,Number of credentials,0,milvus,,, +milvus.rootcoord.ddl_req.count,count,,,,Count of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.bucket,count,,,,Histogram bucket for the latency of each DDL operation,0,milvus,,, +milvus.rootcoord.ddl_req.latency.count,count,,,,Count of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency.sum,count,,millisecond,,Sum of the latencies of DDL operations,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.bucket,count,,,,Histogram bucket for the latencies of each DDL operation in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.count,count,,,,Count of DDL operations in queue,0,milvus,,, +milvus.rootcoord.ddl_req.latency_in_queue.sum,count,,millisecond,,Sum of the latencies of DDL operations in queue,0,milvus,,, +milvus.rootcoord.dml_channel_num,gauge,,,,Number of DML channels,0,milvus,,, +milvus.rootcoord.entity_num,gauge,,,,"Number of entities, clustered by collection and their status(loaded/total)",0,milvus,,, +milvus.rootcoord.force_deny_writing_counter.count,count,,,,The number of times milvus turns into force-deny-writing states,0,milvus,,, +milvus.rootcoord.id_alloc.count,count,,,,Count of ID allocated,0,milvus,,, +milvus.rootcoord.indexed_entity_num,gauge,,,,"Indexed number of entities, clustered by collection, index name and whether it's a vector index",0,milvus,,, +milvus.rootcoord.msgstream_obj_num,gauge,,,,Number of message streams,0,milvus,,, +milvus.rootcoord.num_of_roles,gauge,,,,The number of roles,0,milvus,,, +milvus.rootcoord.partition_num,gauge,,,,Number of partitions,0,milvus,,, +milvus.rootcoord.produce_tt_lag_ms,gauge,,millisecond,,Now time minus time travel per physical channel,0,milvus,,, +milvus.rootcoord.proxy_num,gauge,,,,Number of proxy nodes managered by rootcoord,0,milvus,,, +milvus.rootcoord.qn_mem_high_water_level,gauge,,,,Querynode memory high water level,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.bucket,count,,,,Histogram bucket for the latencies of synchronizing timetick message,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.count,count,,,,Count of synchronizing timetick message events,0,milvus,,, +milvus.rootcoord.sync_timetick_latency.sum,count,,millisecond,,Sum of latencies of synchronizing timetick message events,0,milvus,,, +milvus.rootcoord.timestamp,gauge,,,,Latest timestamp allocated in memory,0,milvus,,, +milvus.rootcoord.timestamp_saved,gauge,,,,Timestamp saved in meta storage,0,milvus,,, +milvus.runtime_info,gauge,,,,Runtime information of Milvus,0,milvus,,, +milvus.search.latency.bucket,count,,,,Histogram bucket for search latencies (ms),0,milvus,,, +milvus.search.latency.count,count,,,,Count of search events,0,milvus,,, +milvus.search.latency.sum,count,,millisecond,,Sum of search latencies (ms),0,milvus,,, +milvus.search.topk.bucket,count,,,,Histogram bucket for search topk,0,milvus,,, +milvus.search.topk.count,count,,,,Count aggregation of search topk,0,milvus,,, +milvus.search.topk.sum,count,,,,Sum aggregation of search topk,0,milvus,,, +milvus.storage.kv_size,gauge,,,,Key-value size stats,0,milvus,,, +milvus.storage.op.count,count,,,,Count of persistent data operation,0,milvus,,, +milvus.storage.request_latency,gauge,,,,Request latency on the client side,0,milvus,,, diff --git a/milvus/pyproject.toml b/milvus/pyproject.toml new file mode 100644 index 0000000000000..7f94dbf91c0d2 --- /dev/null +++ b/milvus/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = [ + "hatchling>=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-milvus" +description = "The Milvus check" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.11" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "milvus", +] +authors = [ + { name = "Datadog", email = "packages@datadoghq.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.11", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=32.6.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [] + +[project.urls] +Source = "https://github.com/DataDog/integrations-core" + +[tool.hatch.version] +path = "datadog_checks/milvus/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/milvus", +] +dev-mode-dirs = [ + ".", +] diff --git a/milvus/tests/__init__.py b/milvus/tests/__init__.py new file mode 100644 index 0000000000000..9103122bf028d --- /dev/null +++ b/milvus/tests/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/milvus/tests/common.py b/milvus/tests/common.py new file mode 100644 index 0000000000000..e2524182310f2 --- /dev/null +++ b/milvus/tests/common.py @@ -0,0 +1,416 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from datadog_checks.dev import get_docker_hostname, get_here + +HERE = get_here() +HOST = get_docker_hostname() +PORT = 9091 + + +def get_fixture_path(filename): + return os.path.join(HERE, 'fixtures', filename) + + +MOCKED_INSTANCE = { + "openmetrics_endpoint": f"http://{HOST}:{PORT}/metrics", + 'tags': ['test:tag'], +} + +COMPOSE_FILE = os.path.join(HERE, 'compose', 'docker-compose.yaml') + +STANDALONE_TEST_METRICS = { + 'milvus.go.gc_duration_seconds.quantile': 'gauge', + 'milvus.go.gc_duration_seconds.sum': 'monotonic_count', + 'milvus.go.gc_duration_seconds.count': 'monotonic_count', + 'milvus.go.goroutines': 'gauge', + 'milvus.go.info': 'gauge', + 'milvus.go.memstats.alloc_bytes': 'gauge', + 'milvus.go.memstats.alloc_bytes.count': 'monotonic_count', + 'milvus.go.memstats.buck_hash_sys_bytes': 'gauge', + 'milvus.go.memstats.frees.count': 'monotonic_count', + 'milvus.go.memstats.gc_sys_bytes': 'gauge', + 'milvus.go.memstats.heap.alloc_bytes': 'gauge', + 'milvus.go.memstats.heap.idle_bytes': 'gauge', + 'milvus.go.memstats.heap.inuse_bytes': 'gauge', + 'milvus.go.memstats.heap.objects': 'gauge', + 'milvus.go.memstats.heap.released_bytes': 'gauge', + 'milvus.go.memstats.heap.sys_bytes': 'gauge', + 'milvus.go.memstats.last_gc_time_seconds': 'gauge', + 'milvus.go.memstats.lookups.count': 'monotonic_count', + 'milvus.go.memstats.mallocs.count': 'monotonic_count', + 'milvus.go.memstats.mcache.inuse_bytes': 'gauge', + 'milvus.go.memstats.mcache.sys_bytes': 'gauge', + 'milvus.go.memstats.mspan.inuse_bytes': 'gauge', + 'milvus.go.memstats.mspan.sys_bytes': 'gauge', + 'milvus.go.memstats.next_gc_bytes': 'gauge', + 'milvus.go.memstats.other_sys_bytes': 'gauge', + 'milvus.go.memstats.stack.inuse_bytes': 'gauge', + 'milvus.go.memstats.stack.sys_bytes': 'gauge', + 'milvus.go.memstats.sys_bytes': 'gauge', + 'milvus.go.threads': 'gauge', + 'milvus.cgo.active_future_total': 'gauge', + 'milvus.cgo.cgo_duration_seconds.sum': 'monotonic_count', + 'milvus.cgo.cgo_duration_seconds.count': 'monotonic_count', + 'milvus.cgo.cgo_duration_seconds.bucket': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.sum': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.count': 'monotonic_count', + 'milvus.cgo.cgo_queue_duration_seconds.bucket': 'monotonic_count', + 'milvus.cgo.running_cgo_call_total': 'gauge', + 'milvus.datacoord.collection_num': 'gauge', + 'milvus.datacoord.consume_datanode_tt_lag_ms': 'gauge', + 'milvus.datacoord.datanode_num': 'gauge', + 'milvus.datacoord.import_tasks': 'gauge', + 'milvus.datacoord.index.task': 'gauge', + 'milvus.datacoord.index.node_num': 'gauge', + 'milvus.datacoord.index.req.count': 'monotonic_count', + 'milvus.datacoord.segment_num': 'gauge', + 'milvus.datacoord.task_execute_max_latency.sum': 'monotonic_count', + 'milvus.datacoord.task_execute_max_latency.count': 'monotonic_count', + 'milvus.datacoord.task_execute_max_latency.bucket': 'monotonic_count', + 'milvus.datanode.autoflush_buffer_op.count': 'monotonic_count', + 'milvus.datanode.consume.bytes.count': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.sum': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.count': 'monotonic_count', + 'milvus.datanode.encode_buffer_latency.bucket': 'monotonic_count', + 'milvus.datanode.flowgraph_num': 'gauge', + 'milvus.datanode.flush.buffer_op.count': 'monotonic_count', + 'milvus.datanode.flush.req.count': 'monotonic_count', + 'milvus.datanode.flushed_data.rows.count': 'monotonic_count', + 'milvus.datanode.flushed_data.size.count': 'monotonic_count', + 'milvus.datanode.msg.rows.count': 'monotonic_count', + 'milvus.datanode.save_latency.sum': 'monotonic_count', + 'milvus.datanode.save_latency.count': 'monotonic_count', + 'milvus.datanode.save_latency.bucket': 'monotonic_count', + 'milvus.flushed_segment_file_num.sum': 'monotonic_count', + 'milvus.flushed_segment_file_num.count': 'monotonic_count', + 'milvus.flushed_segment_file_num.bucket': 'monotonic_count', + 'milvus.indexnode.build_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.build_index_latency.count': 'monotonic_count', + 'milvus.indexnode.build_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.count': 'monotonic_count', + 'milvus.indexnode.encode_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.index.task.count': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.count': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.sum': 'monotonic_count', + 'milvus.indexnode.index.task_latency_in_queue.bucket': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.count': 'monotonic_count', + 'milvus.indexnode.knowhere_build_index_latency.bucket': 'monotonic_count', + 'milvus.indexnode.save_index_latency.sum': 'monotonic_count', + 'milvus.indexnode.save_index_latency.count': 'monotonic_count', + 'milvus.indexnode.save_index_latency.bucket': 'monotonic_count', + 'milvus.meta.kv_size.sum': 'monotonic_count', + 'milvus.meta.kv_size.count': 'monotonic_count', + 'milvus.meta.kv_size.bucket': 'monotonic_count', + 'milvus.meta.op.count': 'monotonic_count', + 'milvus.meta.request_latency.sum': 'monotonic_count', + 'milvus.meta.request_latency.count': 'monotonic_count', + 'milvus.meta.request_latency.bucket': 'monotonic_count', + 'milvus.msg_queue_consumer_num': 'gauge', + 'milvus.msgstream.op.count': 'monotonic_count', + 'milvus.msgstream.request_latency.sum': 'monotonic_count', + 'milvus.msgstream.request_latency.count': 'monotonic_count', + 'milvus.msgstream.request_latency.bucket': 'monotonic_count', + 'milvus.num_node': 'gauge', + 'milvus.proxy.apply.pk_latency.sum': 'monotonic_count', + 'milvus.proxy.apply.pk_latency.count': 'monotonic_count', + 'milvus.proxy.apply.pk_latency.bucket': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.sum': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.count': 'monotonic_count', + 'milvus.proxy.apply.timestamp_latency.bucket': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.sum': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.count': 'monotonic_count', + 'milvus.proxy.assign_segmentID_latency.bucket': 'monotonic_count', + 'milvus.proxy.cache.hit.count': 'monotonic_count', + 'milvus.proxy.cache.update_latency.sum': 'monotonic_count', + 'milvus.proxy.cache.update_latency.count': 'monotonic_count', + 'milvus.proxy.cache.update_latency.bucket': 'monotonic_count', + 'milvus.proxy.delete_vectors.count': 'monotonic_count', + 'milvus.proxy.msgstream_obj_num': 'gauge', + 'milvus.proxy.mutation_send_latency.sum': 'monotonic_count', + 'milvus.proxy.mutation_send_latency.count': 'monotonic_count', + 'milvus.proxy.mutation_send_latency.bucket': 'monotonic_count', + 'milvus.proxy.rate_limit_req.count': 'monotonic_count', + 'milvus.proxy.report_value.count': 'monotonic_count', + 'milvus.proxy.req.count': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.sum': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.count': 'monotonic_count', + 'milvus.proxy.req.in_queue_latency.bucket': 'monotonic_count', + 'milvus.proxy.req.latency.sum': 'monotonic_count', + 'milvus.proxy.req.latency.count': 'monotonic_count', + 'milvus.proxy.req.latency.bucket': 'monotonic_count', + 'milvus.proxy.send_bytes.count': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.decode_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.reduce_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.sum': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.count': 'monotonic_count', + 'milvus.proxy.sq.wait_result_latency.bucket': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.sum': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.count': 'monotonic_count', + 'milvus.proxy.sync_segment_request_length.bucket': 'monotonic_count', + 'milvus.proxy.tt_lag_ms': 'gauge', + 'milvus.querycoord.collection_num': 'gauge', + 'milvus.querycoord.load.latency.sum': 'monotonic_count', + 'milvus.querycoord.load.latency.count': 'monotonic_count', + 'milvus.querycoord.load.latency.bucket': 'monotonic_count', + 'milvus.querycoord.load.req.count': 'monotonic_count', + 'milvus.querycoord.partition_num': 'gauge', + 'milvus.querycoord.querynode_num': 'gauge', + 'milvus.querycoord.release.latency.sum': 'monotonic_count', + 'milvus.querycoord.release.latency.count': 'monotonic_count', + 'milvus.querycoord.release.latency.bucket': 'monotonic_count', + 'milvus.querycoord.release.req.count': 'monotonic_count', + 'milvus.querycoord_task_num': 'gauge', + 'milvus.querynode.apply_bf_latency.sum': 'monotonic_count', + 'milvus.querynode.apply_bf_latency.count': 'monotonic_count', + 'milvus.querynode.apply_bf_latency.bucket': 'monotonic_count', + 'milvus.querynode.collection_num': 'gauge', + 'milvus.querynode.consume.bytes_counter.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.bytes.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.sum': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.disk.cache.evict.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.bytes.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.sum': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.count': 'monotonic_count', + 'milvus.querynode.disk.cache.load.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.disk.cache.load.count': 'monotonic_count', + 'milvus.querynode.disk.used_size': 'gauge', + 'milvus.querynode.dml_vchannel_num': 'gauge', + 'milvus.querynode.execute_bytes_counter.count': 'monotonic_count', + 'milvus.querynode.flowgraph_num': 'gauge', + 'milvus.querynode.forward_delete_latency.sum': 'monotonic_count', + 'milvus.querynode.forward_delete_latency.count': 'monotonic_count', + 'milvus.querynode.forward_delete_latency.bucket': 'monotonic_count', + 'milvus.querynode.load.index_latency.sum': 'monotonic_count', + 'milvus.querynode.load.index_latency.count': 'monotonic_count', + 'milvus.querynode.load.index_latency.bucket': 'monotonic_count', + 'milvus.querynode.load.segment.concurrency': 'gauge', + 'milvus.querynode.load.segment.latency.sum': 'monotonic_count', + 'milvus.querynode.load.segment.latency.count': 'monotonic_count', + 'milvus.querynode.load.segment.latency.bucket': 'monotonic_count', + 'milvus.querynode.process_insert_or_delete_latency.sum': 'monotonic_count', + 'milvus.querynode.process_insert_or_delete_latency.count': 'monotonic_count', + 'milvus.querynode.process_insert_or_delete_latency.bucket': 'monotonic_count', + 'milvus.querynode.read_task.concurrency': 'gauge', + 'milvus.querynode.read_task.ready_len': 'gauge', + 'milvus.querynode.read_task.unsolved_len': 'gauge', + 'milvus.querynode.search.group.nq.sum': 'monotonic_count', + 'milvus.querynode.search.group.nq.count': 'monotonic_count', + 'milvus.querynode.search.group.nq.bucket': 'monotonic_count', + 'milvus.querynode.search.group.size.sum': 'monotonic_count', + 'milvus.querynode.search.group.size.count': 'monotonic_count', + 'milvus.querynode.search.group.size.bucket': 'monotonic_count', + 'milvus.querynode.search.group.topk.sum': 'monotonic_count', + 'milvus.querynode.search.group.topk.count': 'monotonic_count', + 'milvus.querynode.search.group.topk.bucket': 'monotonic_count', + 'milvus.querynode.search.nq.sum': 'monotonic_count', + 'milvus.querynode.search.nq.count': 'monotonic_count', + 'milvus.querynode.search.nq.bucket': 'monotonic_count', + 'milvus.querynode.search.topk.sum': 'monotonic_count', + 'milvus.querynode.search.topk.count': 'monotonic_count', + 'milvus.querynode.search.topk.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.sum': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.sum': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.count': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.global_duration.bucket': 'monotonic_count', + 'milvus.querynode.segment.access.wait_cache.count': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.sum': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.count': 'monotonic_count', + 'milvus.querynode.segment.latency_per_vector.bucket': 'monotonic_count', + 'milvus.querynode.sq.core_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.core_latency.count': 'monotonic_count', + 'milvus.querynode.sq.core_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.sum': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.count': 'monotonic_count', + 'milvus.querynode.sq.queue.latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.count': 'monotonic_count', + 'milvus.querynode.sq.queue.user_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.count': 'monotonic_count', + 'milvus.querynode.sq.reduce_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.req.latency.sum': 'monotonic_count', + 'milvus.querynode.sq.req.latency.count': 'monotonic_count', + 'milvus.querynode.sq.req.latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.count': 'monotonic_count', + 'milvus.querynode.sq.segment_latency.bucket': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.sum': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.count': 'monotonic_count', + 'milvus.querynode.sq.wait_tsafe_latency.bucket': 'monotonic_count', + 'milvus.querynode.wait_processing_msg': 'gauge', + 'milvus.querynode.watch_dml_channel_latency.sum': 'monotonic_count', + 'milvus.querynode.watch_dml_channel_latency.count': 'monotonic_count', + 'milvus.querynode.watch_dml_channel_latency.bucket': 'monotonic_count', + 'milvus.rootcoord.collection_num': 'gauge', + 'milvus.rootcoord.credential_num': 'gauge', + 'milvus.rootcoord.ddl_req.count': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.sum': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.bucket': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.count': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.sum': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.count': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency.bucket': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.sum': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.count': 'monotonic_count', + 'milvus.rootcoord.ddl_req.latency_in_queue.bucket': 'monotonic_count', + 'milvus.rootcoord.dml_channel_num': 'gauge', + 'milvus.rootcoord.entity_num': 'gauge', + 'milvus.rootcoord.force_deny_writing_counter.count': 'monotonic_count', + 'milvus.rootcoord.id_alloc.count': 'monotonic_count', + 'milvus.rootcoord.indexed_entity_num': 'gauge', + 'milvus.rootcoord.msgstream_obj_num': 'gauge', + 'milvus.rootcoord.num_of_roles': 'gauge', + 'milvus.rootcoord.partition_num': 'gauge', + 'milvus.rootcoord.produce_tt_lag_ms': 'gauge', + 'milvus.rootcoord.proxy_num': 'gauge', + 'milvus.rootcoord.qn_mem_high_water_level': 'gauge', + 'milvus.rootcoord.timestamp': 'gauge', + 'milvus.rootcoord.timestamp_saved': 'gauge', + 'milvus.runtime_info': 'gauge', + 'milvus.process.cpu_seconds.count': 'monotonic_count', + 'milvus.process.max_fds': 'gauge', + 'milvus.process.open_fds': 'gauge', + 'milvus.process.resident_memory_bytes': 'gauge', + 'milvus.process.start_time_seconds': 'gauge', + 'milvus.process.virtual_memory.bytes': 'gauge', + 'milvus.process.virtual_memory.max_bytes': 'gauge', + 'milvus.bf_search_cnt.sum': 'monotonic_count', + 'milvus.bf_search_cnt.count': 'monotonic_count', + 'milvus.bf_search_cnt.bucket': 'monotonic_count', + 'milvus.bitset_ratio.sum': 'monotonic_count', + 'milvus.bitset_ratio.count': 'monotonic_count', + 'milvus.bitset_ratio.bucket': 'monotonic_count', + 'milvus.build_latency.sum': 'monotonic_count', + 'milvus.build_latency.count': 'monotonic_count', + 'milvus.build_latency.bucket': 'monotonic_count', + 'milvus.cache_hit_cnt.sum': 'monotonic_count', + 'milvus.cache_hit_cnt.count': 'monotonic_count', + 'milvus.cache_hit_cnt.bucket': 'monotonic_count', + 'milvus.diskann_bitset_ratio.sum': 'monotonic_count', + 'milvus.diskann_bitset_ratio.count': 'monotonic_count', + 'milvus.diskann_bitset_ratio.bucket': 'monotonic_count', + 'milvus.diskann.range_search_iters.sum': 'monotonic_count', + 'milvus.diskann.range_search_iters.count': 'monotonic_count', + 'milvus.diskann.range_search_iters.bucket': 'monotonic_count', + 'milvus.diskann.search_hops.sum': 'monotonic_count', + 'milvus.diskann.search_hops.count': 'monotonic_count', + 'milvus.diskann.search_hops.bucket': 'monotonic_count', + 'milvus.exec_latency.sum': 'monotonic_count', + 'milvus.exec_latency.count': 'monotonic_count', + 'milvus.exec_latency.bucket': 'monotonic_count', + 'milvus.filter.connectivity_ratio.sum': 'monotonic_count', + 'milvus.filter.connectivity_ratio.count': 'monotonic_count', + 'milvus.filter.connectivity_ratio.bucket': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.count': 'monotonic_count', + 'milvus.filter.mv.activated_fields_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.count': 'monotonic_count', + 'milvus.filter.mv.change_base_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.only_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.only_cnt.count': 'monotonic_count', + 'milvus.filter.mv.only_cnt.bucket': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.sum': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.count': 'monotonic_count', + 'milvus.filter.mv.supplement_ep_bool_cnt.bucket': 'monotonic_count', + 'milvus.graph_search_cnt.sum': 'monotonic_count', + 'milvus.graph_search_cnt.count': 'monotonic_count', + 'milvus.graph_search_cnt.bucket': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.sum': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.count': 'monotonic_count', + 'milvus.hnsw.bitset_ratio.bucket': 'monotonic_count', + 'milvus.hnsw.search_hops.sum': 'monotonic_count', + 'milvus.hnsw.search_hops.count': 'monotonic_count', + 'milvus.hnsw.search_hops.bucket': 'monotonic_count', + 'milvus.internal.core_search_latency.sum': 'monotonic_count', + 'milvus.internal.core_search_latency.count': 'monotonic_count', + 'milvus.internal.core_search_latency.bucket': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.sum': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.count': 'monotonic_count', + 'milvus.internal.mmap.allocated_space_bytes.bucket': 'monotonic_count', + 'milvus.internal.mmap.in_used_space_bytes': 'gauge', + 'milvus.internal.storage.kv_size.sum': 'monotonic_count', + 'milvus.internal.storage.kv_size.count': 'monotonic_count', + 'milvus.internal.storage.kv_size.bucket': 'monotonic_count', + 'milvus.internal.storage.load_duration.sum': 'monotonic_count', + 'milvus.internal.storage.load_duration.count': 'monotonic_count', + 'milvus.internal.storage.load_duration.bucket': 'monotonic_count', + 'milvus.internal.storage.op.count': 'monotonic_count', + 'milvus.internal.storage.request_latency.sum': 'monotonic_count', + 'milvus.internal.storage.request_latency.count': 'monotonic_count', + 'milvus.internal.storage.request_latency.bucket': 'monotonic_count', + 'milvus.io_cnt.sum': 'monotonic_count', + 'milvus.io_cnt.count': 'monotonic_count', + 'milvus.io_cnt.bucket': 'monotonic_count', + 'milvus.ivf_search_cnt.sum': 'monotonic_count', + 'milvus.ivf_search_cnt.count': 'monotonic_count', + 'milvus.ivf_search_cnt.bucket': 'monotonic_count', + 'milvus.load_latency.sum': 'monotonic_count', + 'milvus.load_latency.count': 'monotonic_count', + 'milvus.load_latency.bucket': 'monotonic_count', + 'milvus.quant.compute_cnt.sum': 'monotonic_count', + 'milvus.quant.compute_cnt.count': 'monotonic_count', + 'milvus.quant.compute_cnt.bucket': 'monotonic_count', + 'milvus.queue.latency.sum': 'monotonic_count', + 'milvus.queue.latency.count': 'monotonic_count', + 'milvus.queue.latency.bucket': 'monotonic_count', + 'milvus.range_search_latency.sum': 'monotonic_count', + 'milvus.range_search_latency.count': 'monotonic_count', + 'milvus.range_search_latency.bucket': 'monotonic_count', + 'milvus.raw_compute_cnt.sum': 'monotonic_count', + 'milvus.raw_compute_cnt.count': 'monotonic_count', + 'milvus.raw_compute_cnt.bucket': 'monotonic_count', + 'milvus.re_search_cnt.sum': 'monotonic_count', + 'milvus.re_search_cnt.count': 'monotonic_count', + 'milvus.re_search_cnt.bucket': 'monotonic_count', + 'milvus.search.latency.sum': 'monotonic_count', + 'milvus.search.latency.count': 'monotonic_count', + 'milvus.search.latency.bucket': 'monotonic_count', + 'milvus.search.topk.sum': 'monotonic_count', + 'milvus.search.topk.count': 'monotonic_count', + 'milvus.search.topk.bucket': 'monotonic_count', +} + +OTHER_TEST_METRICS = { + 'milvus.datacoord.channel_checkpoint_unix_seconds': 'gauge', + 'milvus.datacoord.stored.binlog_size': 'gauge', + 'milvus.datacoord.stored.index_files_size': 'gauge', + 'milvus.datacoord.stored.rows_num': 'gauge', + 'milvus.datacoord.watched_dml_chanel_num': 'gauge', + 'milvus.datanode.consume.msg_count': 'count', + 'milvus.datanode.consume.tt_lag_ms': 'gauge', + 'milvus.datanode.msg.dispatcher_tt_lag_ms': 'gauge', + 'milvus.querycoord.current_target_checkpoint_unix_seconds': 'gauge', + 'milvus.querynode.consume.msg_count': 'count', + 'milvus.querynode.consume.tt_lag_ms': 'gauge', + 'milvus.querynode.entity.num': 'gauge', + 'milvus.querynode.entity.size': 'gauge', + 'milvus.querynode.msg_dispatcher_tt_lag_ms': 'gauge', + 'milvus.querynode.partition_num': 'gauge', + 'milvus.querynode.segment.num': 'gauge', + 'milvus.querynode.sq.req.count': 'monotonic_count', + 'milvus.rootcoord.sync_timetick_latency.bucket': 'count', + 'milvus.rootcoord.sync_timetick_latency.count': 'count', + 'milvus.rootcoord.timestamp': 'gauge', + 'milvus.storage.kv_size': 'gauge', + 'milvus.storage.op_count': 'count', + 'milvus.storage.request_latency': 'gauge', +} diff --git a/milvus/tests/compose/Dockerfile b/milvus/tests/compose/Dockerfile new file mode 100644 index 0000000000000..9ec85e75bb888 --- /dev/null +++ b/milvus/tests/compose/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9-slim + +# Set working directory +WORKDIR /app + +# Copy requirements file and install dependencies +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY script.py /app/script.py diff --git a/milvus/tests/compose/docker-compose.yaml b/milvus/tests/compose/docker-compose.yaml new file mode 100644 index 0000000000000..d6a4e03eeaf0d --- /dev/null +++ b/milvus/tests/compose/docker-compose.yaml @@ -0,0 +1,41 @@ +services: + milvus: + image: milvusdb/milvus:v2.5.0 + security_opt: + - seccomp:unconfined + environment: + - ETCD_USE_EMBED=true + - ETCD_DATA_DIR=/var/lib/milvus/etcd + - ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml + - COMMON_STORAGETYPE=local + volumes: + - /tmp/volumes/milvus:/var/lib/milvus + - ./embedEtcd.yaml:/milvus/configs/embedEtcd.yaml + - ./user.yaml:/milvus/configs/user.yaml + ports: + - "19530:19530" + - "9091:9091" + - "2379:2379" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + timeout: 20s + start_period: 90s + retries: 3 + command: milvus run standalone + + script-runner: + build: + context: . + dockerfile: Dockerfile + container_name: script-runner + depends_on: + milvus: + condition: service_healthy + volumes: + - ./script.py:/app/script.py + working_dir: /app + command: ["python", "script.py"] + environment: + - MILVUS_HOST=milvus + - MILVUS_PORT=19530 diff --git a/milvus/tests/compose/embedEtcd.yaml b/milvus/tests/compose/embedEtcd.yaml new file mode 100644 index 0000000000000..32954faa8b1fc --- /dev/null +++ b/milvus/tests/compose/embedEtcd.yaml @@ -0,0 +1,5 @@ +listen-client-urls: http://0.0.0.0:2379 +advertise-client-urls: http://0.0.0.0:2379 +quota-backend-bytes: 4294967296 +auto-compaction-mode: revision +auto-compaction-retention: '1000' diff --git a/milvus/tests/compose/requirements.txt b/milvus/tests/compose/requirements.txt new file mode 100644 index 0000000000000..f96eefaacf1ea --- /dev/null +++ b/milvus/tests/compose/requirements.txt @@ -0,0 +1,2 @@ +numpy==1.26.4 +pymilvus==2.5.1 diff --git a/milvus/tests/compose/script.py b/milvus/tests/compose/script.py new file mode 100644 index 0000000000000..c525220925a11 --- /dev/null +++ b/milvus/tests/compose/script.py @@ -0,0 +1,151 @@ +# hello_milvus.py demonstrates the basic operations of PyMilvus, a Python SDK of Milvus. +# 1. connect to Milvus +# 2. create collection +# 3. insert data +# 4. create index +# 5. search, query, and hybrid search on entities +# 6. delete entities by PK +# 7. drop collection +import os +import time + +import numpy as np +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + +num_entities, dim = 3000, 8 + +# Retrieve host and port from environment variables +MILVUS_HOST = os.getenv("MILVUS_HOST", "localhost") +MILVUS_PORT = os.getenv("MILVUS_PORT", "19530") + +################################################################################# +# 1. connect to Milvus +# Add a new connection alias `default` for Milvus server +connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT) + +has = utility.has_collection("hello_milvus") + +################################################################################# +# 2. create collection +# We're going to create a collection with 3 fields. +# +-+------------+------------+------------------+------------------------------+ +# | | field name | field type | other attributes | field description | +# +-+------------+------------+------------------+------------------------------+ +# |1| "pk" | VarChar | is_primary=True | "primary field" | +# | | | | auto_id=False | | +# +-+------------+------------+------------------+------------------------------+ +# |2| "random" | Double | | "a double field" | +# +-+------------+------------+------------------+------------------------------+ +# |3|"embeddings"| FloatVector| dim=8 | "float vector with dim 8" | +# +-+------------+------------+------------------+------------------------------+ +fields = [ + FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=100), + FieldSchema(name="random", dtype=DataType.DOUBLE), + FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim), +] + +schema = CollectionSchema(fields, "hello_milvus is the simplest demo to introduce the APIs") + +hello_milvus = Collection("hello_milvus", schema, consistency_level="Strong") + +################################################################################ +# 3. insert data +# We are going to insert 3000 rows of data into `hello_milvus` +# Data to be inserted must be organized in fields. +# +# The insert() method returns: +# - either automatically generated primary keys by Milvus if auto_id=True in the schema; +# - or the existing primary key field from the entities if auto_id=False in the schema. + +rng = np.random.default_rng(seed=19530) +entities = [ + # provide the pk field because `auto_id` is set to False + [str(i) for i in range(num_entities)], + rng.random(num_entities).tolist(), # field random, only supports list + rng.random((num_entities, dim), np.float32), # field embeddings, supports numpy.ndarray and list +] + +insert_result = hello_milvus.insert(entities) + +row = {"pk": "19530", "random": 0.5, "embeddings": rng.random((1, dim), np.float32)[0]} +hello_milvus.insert(row) + +hello_milvus.flush() + +################################################################################ +# 4. create index +# We are going to create an IVF_FLAT index for hello_milvus collection. +# create_index() can only be applied to `FloatVector` and `BinaryVector` fields. +index = { + "index_type": "IVF_FLAT", + "metric_type": "L2", + "params": {"nlist": 128}, +} + +hello_milvus.create_index("embeddings", index) + +################################################################################ +# 5. search, query, and hybrid search +# After data were inserted into Milvus and indexed, you can perform: +# - search based on vector similarity +# - query based on scalar filtering(boolean, int, etc.) +# - hybrid search based on vector similarity and scalar filtering. +# + +# Before conducting a search or a query, you need to load the data in `hello_milvus` into memory. +hello_milvus.load() + +# ----------------------------------------------------------------------------- +# search based on vector similarity +vectors_to_search = entities[-1][-2:] +search_params = { + "metric_type": "L2", + "params": {"nprobe": 10}, +} + +start_time = time.time() +result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, output_fields=["random"]) +end_time = time.time() + +# ----------------------------------------------------------------------------- +# query based on scalar filtering(boolean, int, etc.) + +start_time = time.time() +result = hello_milvus.query(expr="random > 0.5", output_fields=["random", "embeddings"]) +end_time = time.time() + +# ----------------------------------------------------------------------------- +# pagination +r1 = hello_milvus.query(expr="random > 0.5", limit=4, output_fields=["random"]) +r2 = hello_milvus.query(expr="random > 0.5", offset=1, limit=3, output_fields=["random"]) + + +# ----------------------------------------------------------------------------- +# hybrid search + +start_time = time.time() +result = hello_milvus.search( + vectors_to_search, "embeddings", search_params, limit=3, expr="random > 0.5", output_fields=["random"] +) +end_time = time.time() + +############################################################################### +# 6. delete entities by PK +# You can delete entities by their PK values using boolean expressions. +ids = insert_result.primary_keys + +expr = f'pk in ["{ids[0]}" , "{ids[1]}"]' + +result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"]) + +hello_milvus.delete(expr) + +result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"]) + + +############################################################################### +# 7. drop collection +# Finally, drop the hello_milvus collection +utility.drop_collection("hello_milvus") + +print("Setup complete") diff --git a/milvus/tests/compose/user.yaml b/milvus/tests/compose/user.yaml new file mode 100644 index 0000000000000..8d312694bb2c8 --- /dev/null +++ b/milvus/tests/compose/user.yaml @@ -0,0 +1 @@ +# Extra config to override default milvus.yaml diff --git a/milvus/tests/conftest.py b/milvus/tests/conftest.py new file mode 100644 index 0000000000000..ab0a5dc8b52a7 --- /dev/null +++ b/milvus/tests/conftest.py @@ -0,0 +1,33 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import copy +import logging + +import pytest + +from datadog_checks.dev import docker_run +from datadog_checks.dev.conditions import CheckDockerLogs, CheckEndpoints + +from . import common + +SCRIPT_COMPLETION_STR = "Setup complete" + + +@pytest.fixture(scope='session') +def dd_environment(): + compose_file = common.COMPOSE_FILE + conditions = [ + CheckEndpoints(common.MOCKED_INSTANCE["openmetrics_endpoint"]), + CheckDockerLogs("script-runner", SCRIPT_COMPLETION_STR), + ] + logging.info(conditions) + with docker_run(compose_file, conditions=conditions): + yield { + 'instances': [common.MOCKED_INSTANCE], + } + + +@pytest.fixture +def instance(): + return copy.deepcopy(common.MOCKED_INSTANCE) diff --git a/milvus/tests/fixtures/milvus_payload.txt b/milvus/tests/fixtures/milvus_payload.txt new file mode 100644 index 0000000000000..77895e0a21a2d --- /dev/null +++ b/milvus/tests/fixtures/milvus_payload.txt @@ -0,0 +1,4138 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 2.2583e-05 +go_gc_duration_seconds{quantile="0.25"} 5.5624e-05 +go_gc_duration_seconds{quantile="0.5"} 7e-05 +go_gc_duration_seconds{quantile="0.75"} 0.000267583 +go_gc_duration_seconds{quantile="1"} 0.000976792 +go_gc_duration_seconds_sum 0.003638707 +go_gc_duration_seconds_count 18 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 385 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.22.0"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 8.1514072e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 6.38517176e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.803289e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 2.890576e+06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 5.005784e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 8.1514072e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 1.70811392e+08 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 9.3151232e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 689980 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 1.550336e+08 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 2.63962624e+08 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.7349596816399348e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 3.580556e+06 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 12000 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 15600 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 1.24096e+06 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 1.51776e+06 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.33652608e+08 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.994135e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 4.42368e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 4.42368e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 2.78722872e+08 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 21 +# HELP milvus_build_info Build information of milvus +# TYPE milvus_build_info gauge +milvus_build_info{built="Fri Dec 20 13:51:45 UTC 2024",git_commit="7fe2cd0",version="v2.5.0"} 1 +# HELP milvus_cgo_active_future_total Total number of active futures. +# TYPE milvus_cgo_active_future_total gauge +milvus_cgo_active_future_total{node_id="37"} 0 +# HELP milvus_cgo_cgo_duration_seconds Histogram of cgo call duration in seconds. +# TYPE milvus_cgo_cgo_duration_seconds histogram +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="1.9999999999999998e-05"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_destroy",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_destroy",node_id="37"} 0.000116711 +milvus_cgo_cgo_duration_seconds_count{name="future_destroy",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="1.9999999999999998e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_go_register_ready_callback",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_go_register_ready_callback",node_id="37"} 6.487499999999999e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_go_register_ready_callback",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="9.999999999999999e-06"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="1.9999999999999998e-05"} 6 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="4.9999999999999996e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="9.999999999999999e-05"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.00025"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.0005"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.001"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.002"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="0.01"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="future_leak_and_get",node_id="37",le="+Inf"} 7 +milvus_cgo_cgo_duration_seconds_sum{name="future_leak_and_get",node_id="37"} 9.270699999999999e-05 +milvus_cgo_cgo_duration_seconds_count{name="future_leak_and_get",node_id="37"} 7 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="4.9999999999999996e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="9.999999999999999e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.00025"} 1 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.0005"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.001"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.002"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="0.01"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="retrieve",node_id="37",le="+Inf"} 5 +milvus_cgo_cgo_duration_seconds_sum{name="retrieve",node_id="37"} 0.0015534569999999998 +milvus_cgo_cgo_duration_seconds_count{name="retrieve",node_id="37"} 5 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1.0000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="2.5000000000000004e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="5.000000000000001e-07"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="9.999999999999999e-06"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="1.9999999999999998e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="4.9999999999999996e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="9.999999999999999e-05"} 0 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.00025"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.0005"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.001"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.002"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="0.01"} 2 +milvus_cgo_cgo_duration_seconds_bucket{name="search",node_id="37",le="+Inf"} 2 +milvus_cgo_cgo_duration_seconds_sum{name="search",node_id="37"} 0.000372251 +milvus_cgo_cgo_duration_seconds_count{name="search",node_id="37"} 2 +# HELP milvus_cgo_cgo_queue_duration_seconds Duration of cgo call in queue. +# TYPE milvus_cgo_cgo_queue_duration_seconds histogram +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1e-08"} 0 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1.0000000000000001e-07"} 1 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="2.5000000000000004e-07"} 13 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="5.000000000000001e-07"} 18 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1e-06"} 22 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="9.999999999999999e-06"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="1.9999999999999998e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="4.9999999999999996e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="9.999999999999999e-05"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.00025"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.0005"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.001"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.002"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="0.01"} 28 +milvus_cgo_cgo_queue_duration_seconds_bucket{node_id="37",le="+Inf"} 28 +milvus_cgo_cgo_queue_duration_seconds_sum{node_id="37"} 1.6873e-05 +milvus_cgo_cgo_queue_duration_seconds_count{node_id="37"} 28 +# HELP milvus_cgo_running_cgo_call_total Total number of running cgo calls. +# TYPE milvus_cgo_running_cgo_call_total gauge +milvus_cgo_running_cgo_call_total{node_id="37"} 0 +# HELP milvus_datacoord_collection_num number of collections +# TYPE milvus_datacoord_collection_num gauge +milvus_datacoord_collection_num 0 +# HELP milvus_datacoord_consume_datanode_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_datacoord_consume_datanode_tt_lag_ms gauge +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10",node_id="37"} 487 +milvus_datacoord_consume_datanode_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10_454809251903439155v0",node_id="37"} 487 +# HELP milvus_datacoord_datanode_num number of data nodes +# TYPE milvus_datacoord_datanode_num gauge +milvus_datacoord_datanode_num 1 +# HELP milvus_datacoord_import_tasks the import tasks grouping by type and state +# TYPE milvus_datacoord_import_tasks gauge +milvus_datacoord_import_tasks{import_state="Completed",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Completed",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Failed",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Failed",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="InProgress",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="InProgress",task_type="PreImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Pending",task_type="ImportTask"} 0 +milvus_datacoord_import_tasks{import_state="Pending",task_type="PreImportTask"} 0 +# HELP milvus_datacoord_index_node_num number of IndexNodes managed by IndexCoord +# TYPE milvus_datacoord_index_node_num gauge +milvus_datacoord_index_node_num 1 +# HELP milvus_datacoord_index_req_count number of building index requests +# TYPE milvus_datacoord_index_req_count counter +milvus_datacoord_index_req_count{status="success"} 1 +milvus_datacoord_index_req_count{status="total"} 1 +# HELP milvus_datacoord_index_task_count number of index tasks of each type +# TYPE milvus_datacoord_index_task_count gauge +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808401168564379",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808438016835724",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808487770194226",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808550493389106",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808644260200606",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808703961661599",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808957805920562",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454808985355419955",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454809016083415346",index_task_status="unissued"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="failed"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="finished"} 1 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="in-progress"} 0 +milvus_datacoord_index_task_count{collection_id="454809046208741680",index_task_status="unissued"} 0 +# HELP milvus_datacoord_segment_num number of segments +# TYPE milvus_datacoord_segment_num gauge +milvus_datacoord_segment_num{segment_is_sorted="sorted",segment_level="L1",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="sorted",segment_level="L1",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L0",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L0",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Dropped"} 11 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Flushed"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Flushing"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Growing"} 0 +milvus_datacoord_segment_num{segment_is_sorted="unsorted",segment_level="L1",segment_state="Sealed"} 0 +# HELP milvus_datacoord_task_count number of index tasks of each type +# TYPE milvus_datacoord_task_count gauge +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808401168564379",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808438016835724",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808487770194226",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808550493389106",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808644260200606",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808703961661599",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808957805920562",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454808985355419955",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809016083415346",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809046208741680",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateFailed",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateFinished",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateInProgress",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateInit",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateNone",task_type="JobTypeStatsJob"} 0 +milvus_datacoord_task_count{collection_id="454809251903439155",task_state="JobStateRetry",task_type="JobTypeStatsJob"} 0 +# HELP milvus_datacoord_task_execute_max_latency latency of task execute operation +# TYPE milvus_datacoord_task_execute_max_latency histogram +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="100"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="500"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeIndexJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeIndexJob"} 971 +milvus_datacoord_task_execute_max_latency_count{status="executing",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="100"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="500"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="executing",task_type="JobTypeStatsJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="executing",task_type="JobTypeStatsJob"} 533 +milvus_datacoord_task_execute_max_latency_count{status="executing",task_type="JobTypeStatsJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="100"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="500"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeIndexJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeIndexJob"} 15 +milvus_datacoord_task_execute_max_latency_count{status="pending",task_type="JobTypeIndexJob"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1"} 0 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="100"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="500"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="5000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="10000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="20000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="50000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="100000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="250000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="500000"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="3.6e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="5e+06"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="1e+07"} 1 +milvus_datacoord_task_execute_max_latency_bucket{status="pending",task_type="JobTypeStatsJob",le="+Inf"} 1 +milvus_datacoord_task_execute_max_latency_sum{status="pending",task_type="JobTypeStatsJob"} 5 +milvus_datacoord_task_execute_max_latency_count{status="pending",task_type="JobTypeStatsJob"} 1 +# HELP milvus_datanode_autoflush_buffer_op_count count of auto flush buffer operations +# TYPE milvus_datanode_autoflush_buffer_op_count counter +milvus_datanode_autoflush_buffer_op_count{node_id="37",segment_level="L1",status="success"} 1 +# HELP milvus_datanode_consume_bytes_count +# TYPE milvus_datanode_consume_bytes_count counter +milvus_datanode_consume_bytes_count{msg_type="delete",node_id="37"} 241 +milvus_datanode_consume_bytes_count{msg_type="insert",node_id="37"} 191574 +# HELP milvus_datanode_encode_buffer_latency latency of encode buffer data +# TYPE milvus_datanode_encode_buffer_latency histogram +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="1"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="2"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="4"} 0 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="8"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="16"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="32"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="64"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="128"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="256"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="512"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="1024"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="2048"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="4096"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="8192"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="16384"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="32768"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="65536"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="131072"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L0",le="+Inf"} 1 +milvus_datanode_encode_buffer_latency_sum{node_id="37",segment_level="L0"} 6 +milvus_datanode_encode_buffer_latency_count{node_id="37",segment_level="L0"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="1"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="2"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="4"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="8"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="16"} 1 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="32"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="64"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="128"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="256"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="512"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="1024"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="2048"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="4096"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="8192"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="16384"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="32768"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="65536"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="131072"} 2 +milvus_datanode_encode_buffer_latency_bucket{node_id="37",segment_level="L1",le="+Inf"} 2 +milvus_datanode_encode_buffer_latency_sum{node_id="37",segment_level="L1"} 20 +milvus_datanode_encode_buffer_latency_count{node_id="37",segment_level="L1"} 2 +# HELP milvus_datanode_flowgraph_num number of flowgraphs +# TYPE milvus_datanode_flowgraph_num gauge +milvus_datanode_flowgraph_num{node_id="37"} 0 +# HELP milvus_datanode_flush_buffer_op_count count of flush buffer operations +# TYPE milvus_datanode_flush_buffer_op_count counter +milvus_datanode_flush_buffer_op_count{node_id="37",segment_level="L0",status="success"} 1 +milvus_datanode_flush_buffer_op_count{node_id="37",segment_level="L1",status="success"} 2 +# HELP milvus_datanode_flush_req_count count of flush request +# TYPE milvus_datanode_flush_req_count counter +milvus_datanode_flush_req_count{node_id="37",status="success"} 1 +milvus_datanode_flush_req_count{node_id="37",status="total"} 3 +# HELP milvus_datanode_flushed_data_rows num of rows flushed to storage +# TYPE milvus_datanode_flushed_data_rows counter +milvus_datanode_flushed_data_rows{data_source="streaming",node_id="37"} 3001 +# HELP milvus_datanode_flushed_data_size byte size of data flushed to storage +# TYPE milvus_datanode_flushed_data_size counter +milvus_datanode_flushed_data_size{data_source="streaming",node_id="37",segment_level="L0"} 620 +milvus_datanode_flushed_data_size{data_source="streaming",node_id="37",segment_level="L1"} 226983 +# HELP milvus_datanode_msg_rows_count count of rows consumed from msgStream +# TYPE milvus_datanode_msg_rows_count counter +milvus_datanode_msg_rows_count{msg_type="delete",node_id="37"} 2 +milvus_datanode_msg_rows_count{msg_type="insert",node_id="37"} 3001 +# HELP milvus_datanode_save_latency latency of saving flush data to storage +# TYPE milvus_datanode_save_latency histogram +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="10"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="100"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="200"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="400"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="1000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="10000"} 1 +milvus_datanode_save_latency_bucket{msg_type="L0",node_id="37",le="+Inf"} 1 +milvus_datanode_save_latency_sum{msg_type="L0",node_id="37"} 5 +milvus_datanode_save_latency_count{msg_type="L0",node_id="37"} 1 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="0"} 0 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="10"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="100"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="200"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="400"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="1000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="10000"} 2 +milvus_datanode_save_latency_bucket{msg_type="L1",node_id="37",le="+Inf"} 2 +milvus_datanode_save_latency_sum{msg_type="L1",node_id="37"} 14 +milvus_datanode_save_latency_count{msg_type="L1",node_id="37"} 2 +# HELP milvus_flushed_segment_file_num the num of files for flushed segment +# TYPE milvus_flushed_segment_file_num histogram +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="2"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="4"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="131072"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="delete_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="delete_file"} 0 +milvus_flushed_segment_file_num_count{segment_file_type="delete_file"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="2"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="4"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="8"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="16"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="32"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="64"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="128"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="256"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="512"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="1024"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="2048"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="4096"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="8192"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="16384"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="32768"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="65536"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="131072"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="index_file",le="+Inf"} 11 +milvus_flushed_segment_file_num_sum{segment_file_type="index_file"} 11 +milvus_flushed_segment_file_num_count{segment_file_type="index_file"} 11 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="2"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="4"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="131072"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="insert_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="insert_file"} 5 +milvus_flushed_segment_file_num_count{segment_file_type="insert_file"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1"} 0 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="2"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="4"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="8"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="16"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="32"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="64"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="128"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="256"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="512"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="1024"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="2048"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="4096"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="8192"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="16384"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="32768"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="65536"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="131072"} 1 +milvus_flushed_segment_file_num_bucket{segment_file_type="stat_file",le="+Inf"} 1 +milvus_flushed_segment_file_num_sum{segment_file_type="stat_file"} 2 +milvus_flushed_segment_file_num_count{segment_file_type="stat_file"} 1 +# HELP milvus_indexnode_build_index_latency latency of build index for segment +# TYPE milvus_indexnode_build_index_latency histogram +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_build_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_build_index_latency_sum{node_id="37"} 0.033203625 +milvus_indexnode_build_index_latency_count{node_id="37"} 1 +# HELP milvus_indexnode_encode_index_latency latency of encoding the index file +# TYPE milvus_indexnode_encode_index_latency histogram +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_encode_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_encode_index_latency_sum{node_id="37"} 0.007635709 +milvus_indexnode_encode_index_latency_count{node_id="37"} 1 +# HELP milvus_indexnode_index_task_count number of tasks that index node received +# TYPE milvus_indexnode_index_task_count counter +milvus_indexnode_index_task_count{node_id="37",status="success"} 1 +# HELP milvus_indexnode_index_task_latency_in_queue latency of index task in queue +# TYPE milvus_indexnode_index_task_latency_in_queue histogram +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="1"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="2"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="4"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="8"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="16"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="32"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="64"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="128"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="256"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="512"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="1024"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="2048"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="4096"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="8192"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="16384"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="32768"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="65536"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="131072"} 1 +milvus_indexnode_index_task_latency_in_queue_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_index_task_latency_in_queue_sum{node_id="37"} 0 +milvus_indexnode_index_task_latency_in_queue_count{node_id="37"} 1 +# HELP milvus_indexnode_knowhere_build_index_latency latency of building the index by knowhere +# TYPE milvus_indexnode_knowhere_build_index_latency histogram +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.001"} 0 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_knowhere_build_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_knowhere_build_index_latency_sum{node_id="37"} 0.025299416 +milvus_indexnode_knowhere_build_index_latency_count{node_id="37"} 1 +# HELP milvus_indexnode_save_index_latency latency of saving the index file +# TYPE milvus_indexnode_save_index_latency histogram +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.001"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="0.5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="1"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="5"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="10"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="20"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="50"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="100"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="250"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="500"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="3600"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_indexnode_save_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_indexnode_save_index_latency_sum{node_id="37"} 6.0041e-05 +milvus_indexnode_save_index_latency_count{node_id="37"} 1 +# HELP milvus_meta_kv_size kv size stats +# TYPE milvus_meta_kv_size histogram +milvus_meta_kv_size_bucket{meta_op_type="get",le="1"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="2"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="4"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="8"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="16"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="32"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="64"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="128"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="256"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="512"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="1024"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="2048"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="4096"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="8192"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="16384"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="32768"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="65536"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="131072"} 89 +milvus_meta_kv_size_bucket{meta_op_type="get",le="+Inf"} 89 +milvus_meta_kv_size_sum{meta_op_type="get"} -313 +milvus_meta_kv_size_count{meta_op_type="get"} 89 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="2"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="4"} 1 +milvus_meta_kv_size_bucket{meta_op_type="put",le="8"} 39 +milvus_meta_kv_size_bucket{meta_op_type="put",le="16"} 40 +milvus_meta_kv_size_bucket{meta_op_type="put",le="32"} 40 +milvus_meta_kv_size_bucket{meta_op_type="put",le="64"} 50 +milvus_meta_kv_size_bucket{meta_op_type="put",le="128"} 60 +milvus_meta_kv_size_bucket{meta_op_type="put",le="256"} 63 +milvus_meta_kv_size_bucket{meta_op_type="put",le="512"} 72 +milvus_meta_kv_size_bucket{meta_op_type="put",le="1024"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="2048"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="4096"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="8192"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="16384"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="32768"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="65536"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="131072"} 76 +milvus_meta_kv_size_bucket{meta_op_type="put",le="+Inf"} 76 +milvus_meta_kv_size_sum{meta_op_type="put"} 8526 +milvus_meta_kv_size_count{meta_op_type="put"} 76 +# HELP milvus_meta_op_count count of meta operation +# TYPE milvus_meta_op_count counter +milvus_meta_op_count{meta_op_type="get",status="success"} 63 +milvus_meta_op_count{meta_op_type="get",status="total"} 63 +milvus_meta_op_count{meta_op_type="put",status="success"} 50 +milvus_meta_op_count{meta_op_type="put",status="total"} 50 +milvus_meta_op_count{meta_op_type="remove",status="success"} 6 +milvus_meta_op_count{meta_op_type="remove",status="total"} 6 +milvus_meta_op_count{meta_op_type="txn",status="success"} 26 +milvus_meta_op_count{meta_op_type="txn",status="total"} 26 +# HELP milvus_meta_request_latency request latency on the client side +# TYPE milvus_meta_request_latency histogram +milvus_meta_request_latency_bucket{meta_op_type="get",le="1"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="2"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="4"} 62 +milvus_meta_request_latency_bucket{meta_op_type="get",le="8"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="16"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="32"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="64"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="128"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="256"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="512"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="1024"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="2048"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="4096"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="8192"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="16384"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="32768"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="65536"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="131072"} 63 +milvus_meta_request_latency_bucket{meta_op_type="get",le="+Inf"} 63 +milvus_meta_request_latency_sum{meta_op_type="get"} 7 +milvus_meta_request_latency_count{meta_op_type="get"} 63 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1"} 19 +milvus_meta_request_latency_bucket{meta_op_type="put",le="2"} 35 +milvus_meta_request_latency_bucket{meta_op_type="put",le="4"} 45 +milvus_meta_request_latency_bucket{meta_op_type="put",le="8"} 49 +milvus_meta_request_latency_bucket{meta_op_type="put",le="16"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="32"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="64"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="128"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="256"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="512"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="1024"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="2048"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="4096"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="8192"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="16384"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="32768"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="65536"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="131072"} 50 +milvus_meta_request_latency_bucket{meta_op_type="put",le="+Inf"} 50 +milvus_meta_request_latency_sum{meta_op_type="put"} 106 +milvus_meta_request_latency_count{meta_op_type="put"} 50 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="2"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="4"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="8"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="16"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="32"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="64"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="128"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="256"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="512"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="1024"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="2048"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="4096"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="8192"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="16384"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="32768"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="65536"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="131072"} 6 +milvus_meta_request_latency_bucket{meta_op_type="remove",le="+Inf"} 6 +milvus_meta_request_latency_sum{meta_op_type="remove"} 0 +milvus_meta_request_latency_count{meta_op_type="remove"} 6 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1"} 22 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="2"} 25 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="4"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="8"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="16"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="32"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="64"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="128"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="256"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="512"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="1024"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="2048"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="4096"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="8192"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="16384"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="32768"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="65536"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="131072"} 26 +milvus_meta_request_latency_bucket{meta_op_type="txn",le="+Inf"} 26 +milvus_meta_request_latency_sum{meta_op_type="txn"} 14 +milvus_meta_request_latency_count{meta_op_type="txn"} 26 +# HELP milvus_msg_queue_consumer_num number of consumers +# TYPE milvus_msg_queue_consumer_num gauge +milvus_msg_queue_consumer_num{node_id="37",role_name="standalone"} 0 +# HELP milvus_msgstream_op_count count of stream message operation +# TYPE milvus_msgstream_op_count counter +milvus_msgstream_op_count{message_op_type="create_consumer",status="success"} 2 +milvus_msgstream_op_count{message_op_type="create_consumer",status="total"} 2 +milvus_msgstream_op_count{message_op_type="create_producer",status="success"} 18 +milvus_msgstream_op_count{message_op_type="create_producer",status="total"} 18 +milvus_msgstream_op_count{message_op_type="produce",status="success"} 3594 +milvus_msgstream_op_count{message_op_type="produce",status="total"} 3594 +# HELP milvus_msgstream_request_latency request latency on the client side +# TYPE milvus_msgstream_request_latency histogram +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="2"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="4"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="8"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="16"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="32"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="64"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="128"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="256"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="512"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="1024"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="2048"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="4096"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="8192"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="16384"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="32768"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="65536"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="131072"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_consumer",le="+Inf"} 2 +milvus_msgstream_request_latency_sum{message_op_type="create_consumer"} 0 +milvus_msgstream_request_latency_count{message_op_type="create_consumer"} 2 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="2"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="4"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="8"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="16"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="32"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="64"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="128"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="256"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="512"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="1024"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="2048"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="4096"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="8192"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="16384"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="32768"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="65536"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="131072"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="create_producer",le="+Inf"} 18 +milvus_msgstream_request_latency_sum{message_op_type="create_producer"} 0 +milvus_msgstream_request_latency_count{message_op_type="create_producer"} 18 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1"} 781 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="2"} 1620 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="4"} 2887 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="8"} 3438 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="16"} 3583 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="32"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="64"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="128"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="256"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="512"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="1024"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="2048"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="4096"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="8192"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="16384"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="32768"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="65536"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="131072"} 3594 +milvus_msgstream_request_latency_bucket{message_op_type="produce",le="+Inf"} 3594 +milvus_msgstream_request_latency_sum{message_op_type="produce"} 11317 +milvus_msgstream_request_latency_count{message_op_type="produce"} 3594 +# HELP milvus_num_node number of nodes and coordinates +# TYPE milvus_num_node gauge +milvus_num_node{node_id="37",role_name="datacoord"} 1 +milvus_num_node{node_id="37",role_name="datanode"} 1 +milvus_num_node{node_id="37",role_name="indexnode"} 1 +milvus_num_node{node_id="37",role_name="proxy"} 1 +milvus_num_node{node_id="37",role_name="querycoord"} 1 +milvus_num_node{node_id="37",role_name="querynode"} 1 +milvus_num_node{node_id="37",role_name="rootcoord"} 1 +# HELP milvus_proxy_apply_pk_latency latency that apply primary key +# TYPE milvus_proxy_apply_pk_latency histogram +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="1"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="2"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="4"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="8"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="16"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="32"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="64"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="128"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="256"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="512"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="1024"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="2048"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="4096"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="8192"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="16384"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="32768"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="65536"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="131072"} 2 +milvus_proxy_apply_pk_latency_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_apply_pk_latency_sum{node_id="37"} 0 +milvus_proxy_apply_pk_latency_count{node_id="37"} 2 +# HELP milvus_proxy_apply_timestamp_latency latency that proxy apply timestamp +# TYPE milvus_proxy_apply_timestamp_latency histogram +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="1"} 357 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="2"} 523 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="4"} 561 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="8"} 577 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="16"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="32"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="64"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="128"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="256"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="512"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="1024"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="2048"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="4096"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="8192"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="16384"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="32768"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="65536"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="131072"} 578 +milvus_proxy_apply_timestamp_latency_bucket{node_id="37",le="+Inf"} 578 +milvus_proxy_apply_timestamp_latency_sum{node_id="37"} 750 +milvus_proxy_apply_timestamp_latency_count{node_id="37"} 578 +# HELP milvus_proxy_assign_segmentID_latency latency that proxy get segmentID from dataCoord +# TYPE milvus_proxy_assign_segmentID_latency histogram +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="1"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="2"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="4"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="8"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="16"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="32"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="64"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="128"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="256"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="512"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="1024"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="2048"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="4096"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="8192"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="16384"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="32768"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="65536"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="131072"} 2 +milvus_proxy_assign_segmentID_latency_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_assign_segmentID_latency_sum{node_id="37"} 1 +milvus_proxy_assign_segmentID_latency_count{node_id="37"} 2 +# HELP milvus_proxy_cache_hit_count count of cache hits/miss +# TYPE milvus_proxy_cache_hit_count counter +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="hit",node_id="37"} 51 +milvus_proxy_cache_hit_count{cache_name="GetCollectionID",cache_state="miss",node_id="37"} 3 +milvus_proxy_cache_hit_count{cache_name="GetCollectionInfo",cache_state="hit",node_id="37"} 19 +milvus_proxy_cache_hit_count{cache_name="GetCollectionSchema",cache_state="hit",node_id="37"} 27 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="hit",node_id="37"} 6 +milvus_proxy_cache_hit_count{cache_name="GetShards",cache_state="miss",node_id="37"} 2 +# HELP milvus_proxy_cache_update_latency latency that proxy update cache when cache miss +# TYPE milvus_proxy_cache_update_latency histogram +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="1"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="2"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="4"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="8"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="16"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="32"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="64"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="128"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="256"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="512"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="1024"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="2048"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="4096"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="8192"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="16384"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="32768"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="65536"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="131072"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetCollectionID",node_id="37",le="+Inf"} 2 +milvus_proxy_cache_update_latency_sum{cache_name="GetCollectionID",node_id="37"} 1 +milvus_proxy_cache_update_latency_count{cache_name="GetCollectionID",node_id="37"} 2 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="1"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="2"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="4"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="8"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="16"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="32"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="64"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="128"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="256"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="512"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="1024"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="2048"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="4096"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="8192"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="16384"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="32768"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="65536"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="131072"} 1 +milvus_proxy_cache_update_latency_bucket{cache_name="GetShards",node_id="37",le="+Inf"} 1 +milvus_proxy_cache_update_latency_sum{cache_name="GetShards",node_id="37"} 1 +milvus_proxy_cache_update_latency_count{cache_name="GetShards",node_id="37"} 1 +# HELP milvus_proxy_delete_vectors_count counter of vectors successfully deleted +# TYPE milvus_proxy_delete_vectors_count counter +milvus_proxy_delete_vectors_count{collection_name="hello_milvus",db_name="default",node_id="37"} 2 +# HELP milvus_proxy_msgstream_obj_num number of MsgStream objects per physical channel +# TYPE milvus_proxy_msgstream_obj_num gauge +milvus_proxy_msgstream_obj_num{channel_name="by-dev-rootcoord-dml_10",node_id="37"} 0 +# HELP milvus_proxy_mutation_send_latency latency that proxy send insert request to MsgStream +# TYPE milvus_proxy_mutation_send_latency histogram +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="100"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="500"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="5000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="10000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="20000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="50000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="100000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="250000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="500000"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="3.6e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="5e+06"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="1e+07"} 2 +milvus_proxy_mutation_send_latency_bucket{msg_type="insert",node_id="37",le="+Inf"} 2 +milvus_proxy_mutation_send_latency_sum{msg_type="insert",node_id="37"} 1 +milvus_proxy_mutation_send_latency_count{msg_type="insert",node_id="37"} 2 +# HELP milvus_proxy_rate_limit_req_count count of operation executed +# TYPE milvus_proxy_rate_limit_req_count counter +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="37",status="success"} 23 +milvus_proxy_rate_limit_req_count{msg_type="DDLCollection",node_id="37",status="total"} 23 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLFlush",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DDLIndex",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="37",status="success"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLDelete",node_id="37",status="total"} 1 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="37",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DMLInsert",node_id="37",status="total"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="37",status="success"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLQuery",node_id="37",status="total"} 5 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="37",status="success"} 2 +milvus_proxy_rate_limit_req_count{msg_type="DQLSearch",node_id="37",status="total"} 2 +# HELP milvus_proxy_report_value report value about the request +# TYPE milvus_proxy_report_value counter +milvus_proxy_report_value{db_name="default",msg_type="delete",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="insert",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="query",node_id="37",username=""} 0 +milvus_proxy_report_value{db_name="default",msg_type="search",node_id="37",username=""} 0 +# HELP milvus_proxy_req_count count of operation executed +# TYPE milvus_proxy_req_count counter +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="37",status="success"} 1 +milvus_proxy_req_count{collection_name="",db_name="default",function_name="Flush",node_id="37",status="total"} 1 +milvus_proxy_req_count{collection_name="hello_milvus",db_name="default",function_name="DropCollection",node_id="37",status="success"} 1 +# HELP milvus_proxy_req_in_queue_latency latency which request waits in the queue +# TYPE milvus_proxy_req_in_queue_latency histogram +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="CreateIndex",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="CreateIndex",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="CreateIndex",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Delete",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Delete",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Delete",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="1"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="2"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="4"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="8"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="16"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="32"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="64"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="128"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="256"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="512"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="1024"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="2048"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="4096"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="8192"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="16384"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="32768"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="65536"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="131072"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeCollection",node_id="37",le="+Inf"} 3 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeCollection",node_id="37"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="1"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="2"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="4"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="8"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="16"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="32"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="64"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="128"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="256"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="512"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="1024"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="2048"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="4096"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="8192"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="16384"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="32768"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="65536"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="131072"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DescribeIndex",node_id="37",le="+Inf"} 3 +milvus_proxy_req_in_queue_latency_sum{function_name="DescribeIndex",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DescribeIndex",node_id="37"} 3 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="DropCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="DropCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="DropCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Flush",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="Flush",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Flush",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="1"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="4"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="8"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="16"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="32"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="64"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="128"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="256"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="512"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="1024"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="2048"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="4096"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="8192"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="16384"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="32768"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="65536"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="131072"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Insert",node_id="37",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Insert",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Insert",node_id="37"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="1"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="2"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="4"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="8"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="16"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="32"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="64"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="128"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="256"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="512"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="LoadCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_in_queue_latency_sum{function_name="LoadCollection",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="LoadCollection",node_id="37"} 1 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="1"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="2"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="4"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="8"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="16"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="32"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="64"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="128"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="256"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="512"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="1024"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="2048"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="4096"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="8192"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="16384"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="32768"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="65536"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="131072"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Retrieve",node_id="37",le="+Inf"} 5 +milvus_proxy_req_in_queue_latency_sum{function_name="Retrieve",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Retrieve",node_id="37"} 5 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="1"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="2"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="4"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="8"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="16"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="32"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="64"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="128"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="256"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="512"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="1024"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="2048"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="4096"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="8192"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="16384"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="32768"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="65536"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="131072"} 2 +milvus_proxy_req_in_queue_latency_bucket{function_name="Search",node_id="37",le="+Inf"} 2 +milvus_proxy_req_in_queue_latency_sum{function_name="Search",node_id="37"} 0 +milvus_proxy_req_in_queue_latency_count{function_name="Search",node_id="37"} 2 +# HELP milvus_proxy_req_latency latency of each request +# TYPE milvus_proxy_req_latency histogram +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="4"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateCollection",node_id="37"} 4 +milvus_proxy_req_latency_count{function_name="CreateCollection",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="CreateIndex",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="CreateIndex",node_id="37"} 7 +milvus_proxy_req_latency_count{function_name="CreateIndex",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="1"} 2 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="2"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="4"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="8"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="16"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="32"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="64"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="128"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="256"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="512"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="1024"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="2048"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="4096"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="8192"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="16384"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="32768"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="65536"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="131072"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeCollection",node_id="37",le="+Inf"} 3 +milvus_proxy_req_latency_sum{function_name="DescribeCollection",node_id="37"} 2 +milvus_proxy_req_latency_count{function_name="DescribeCollection",node_id="37"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="2"} 2 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="4"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="8"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="16"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="32"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="64"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="128"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="256"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="512"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="1024"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="2048"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="4096"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="8192"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="16384"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="32768"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="65536"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="131072"} 3 +milvus_proxy_req_latency_bucket{function_name="DescribeIndex",node_id="37",le="+Inf"} 3 +milvus_proxy_req_latency_sum{function_name="DescribeIndex",node_id="37"} 8 +milvus_proxy_req_latency_count{function_name="DescribeIndex",node_id="37"} 3 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="DropCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="DropCollection",node_id="37"} 6 +milvus_proxy_req_latency_count{function_name="DropCollection",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="8"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="16"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="32"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="64"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="128"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="256"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="512"} 0 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="Flush",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="Flush",node_id="37"} 619 +milvus_proxy_req_latency_count{function_name="Flush",node_id="37"} 1 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="1"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="2"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="4"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="8"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="16"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="32"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="64"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="128"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="256"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="512"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="1024"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="2048"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="4096"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="8192"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="16384"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="32768"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="65536"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="131072"} 6 +milvus_proxy_req_latency_bucket{function_name="GetLoadingProgress",node_id="37",le="+Inf"} 6 +milvus_proxy_req_latency_sum{function_name="GetLoadingProgress",node_id="37"} 2 +milvus_proxy_req_latency_count{function_name="GetLoadingProgress",node_id="37"} 6 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="1"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="2"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="4"} 0 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="8"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="16"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="32"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="64"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="128"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="256"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="512"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="1024"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="2048"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="4096"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="8192"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="16384"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="32768"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="65536"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="131072"} 1 +milvus_proxy_req_latency_bucket{function_name="LoadCollection",node_id="37",le="+Inf"} 1 +milvus_proxy_req_latency_sum{function_name="LoadCollection",node_id="37"} 7 +milvus_proxy_req_latency_count{function_name="LoadCollection",node_id="37"} 1 +# HELP milvus_proxy_search_sparse_num_non_zeros the number of non-zeros in each sparse search task +# TYPE milvus_proxy_search_sparse_num_non_zeros histogram +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="1"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="2"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="4"} 0 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="8"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="16"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="32"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="64"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="128"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="256"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="512"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="1024"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="2048"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="4096"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="8192"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="16384"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="32768"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="65536"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="131072"} 2 +milvus_proxy_search_sparse_num_non_zeros_bucket{collection_name="hello_milvus",node_id="37",le="+Inf"} 2 +milvus_proxy_search_sparse_num_non_zeros_sum{collection_name="hello_milvus",node_id="37"} 16 +milvus_proxy_search_sparse_num_non_zeros_count{collection_name="hello_milvus",node_id="37"} 2 +# HELP milvus_proxy_send_bytes_count count of bytes sent back to sdk +# TYPE milvus_proxy_send_bytes_count counter +milvus_proxy_send_bytes_count{node_id="37"} 70813 +# HELP milvus_proxy_sq_decode_result_latency latency that proxy decodes the search result +# TYPE milvus_proxy_sq_decode_result_latency histogram +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="1"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="2"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_decode_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_decode_result_latency_sum{node_id="37",query_type="query"} 0 +milvus_proxy_sq_decode_result_latency_count{node_id="37",query_type="query"} 5 +# HELP milvus_proxy_sq_reduce_result_latency latency that proxy reduces search result +# TYPE milvus_proxy_sq_reduce_result_latency histogram +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="1"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="2"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_reduce_result_latency_sum{node_id="37",query_type="query"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="37",query_type="query"} 5 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="1"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="2"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="4"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="8"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="16"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="32"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="64"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="128"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="256"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_proxy_sq_reduce_result_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_reduce_result_latency_sum{node_id="37",query_type="search"} 0 +milvus_proxy_sq_reduce_result_latency_count{node_id="37",query_type="search"} 2 +# HELP milvus_proxy_sq_wait_result_latency latency that proxy waits for the result +# TYPE milvus_proxy_sq_wait_result_latency histogram +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="2"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="4"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="8"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="16"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="32"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="64"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="128"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="256"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_proxy_sq_wait_result_latency_sum{node_id="37",query_type="query"} 1478 +milvus_proxy_sq_wait_result_latency_count{node_id="37",query_type="query"} 5 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="2"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="4"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="8"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="16"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="32"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="64"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="128"} 0 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="256"} 1 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_proxy_sq_wait_result_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_proxy_sq_wait_result_latency_sum{node_id="37",query_type="search"} 610 +milvus_proxy_sq_wait_result_latency_count{node_id="37",query_type="search"} 2 +# HELP milvus_proxy_sync_segment_request_length the length of SegmentIDRequests when assigning segments for insert +# TYPE milvus_proxy_sync_segment_request_length histogram +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="1"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="2"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="4"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="8"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="16"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="32"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="64"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="128"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="256"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="512"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="1024"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="2048"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="4096"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="8192"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="16384"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="32768"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="65536"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="131072"} 2 +milvus_proxy_sync_segment_request_length_bucket{node_id="37",le="+Inf"} 2 +milvus_proxy_sync_segment_request_length_sum{node_id="37"} 2 +milvus_proxy_sync_segment_request_length_count{node_id="37"} 2 +# HELP milvus_proxy_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_proxy_tt_lag_ms gauge +milvus_proxy_tt_lag_ms{channel_name="default",node_id="37"} 36 +# HELP milvus_querycoord_collection_num number of collections +# TYPE milvus_querycoord_collection_num gauge +milvus_querycoord_collection_num 0 +# HELP milvus_querycoord_load_latency latency of load the entire collection +# TYPE milvus_querycoord_load_latency histogram +milvus_querycoord_load_latency_bucket{le="0"} 0 +milvus_querycoord_load_latency_bucket{le="500"} 0 +milvus_querycoord_load_latency_bucket{le="1000"} 2 +milvus_querycoord_load_latency_bucket{le="2000"} 2 +milvus_querycoord_load_latency_bucket{le="5000"} 2 +milvus_querycoord_load_latency_bucket{le="10000"} 2 +milvus_querycoord_load_latency_bucket{le="20000"} 2 +milvus_querycoord_load_latency_bucket{le="50000"} 2 +milvus_querycoord_load_latency_bucket{le="60000"} 2 +milvus_querycoord_load_latency_bucket{le="300000"} 2 +milvus_querycoord_load_latency_bucket{le="600000"} 2 +milvus_querycoord_load_latency_bucket{le="1.8e+06"} 2 +milvus_querycoord_load_latency_bucket{le="+Inf"} 2 +milvus_querycoord_load_latency_sum 1996 +milvus_querycoord_load_latency_count 2 +# HELP milvus_querycoord_load_req_count count of load request +# TYPE milvus_querycoord_load_req_count counter +milvus_querycoord_load_req_count{status="success"} 1 +milvus_querycoord_load_req_count{status="total"} 1 +# HELP milvus_querycoord_partition_num number of partitions +# TYPE milvus_querycoord_partition_num gauge +milvus_querycoord_partition_num 0 +# HELP milvus_querycoord_querynode_num number of QueryNodes managered by QueryCoord +# TYPE milvus_querycoord_querynode_num gauge +milvus_querycoord_querynode_num 1 +# HELP milvus_querycoord_release_latency latency of release request +# TYPE milvus_querycoord_release_latency histogram +milvus_querycoord_release_latency_bucket{le="0"} 10 +milvus_querycoord_release_latency_bucket{le="5"} 10 +milvus_querycoord_release_latency_bucket{le="10"} 10 +milvus_querycoord_release_latency_bucket{le="20"} 10 +milvus_querycoord_release_latency_bucket{le="40"} 10 +milvus_querycoord_release_latency_bucket{le="100"} 10 +milvus_querycoord_release_latency_bucket{le="200"} 10 +milvus_querycoord_release_latency_bucket{le="400"} 10 +milvus_querycoord_release_latency_bucket{le="1000"} 11 +milvus_querycoord_release_latency_bucket{le="10000"} 11 +milvus_querycoord_release_latency_bucket{le="+Inf"} 11 +milvus_querycoord_release_latency_sum 606 +milvus_querycoord_release_latency_count 11 +# HELP milvus_querycoord_release_req_count count of release request +# TYPE milvus_querycoord_release_req_count counter +milvus_querycoord_release_req_count{status="success"} 1 +milvus_querycoord_release_req_count{status="total"} 1 +# HELP milvus_querycoord_replica_ro_node_total total read only node number of replica +# TYPE milvus_querycoord_replica_ro_node_total gauge +milvus_querycoord_replica_ro_node_total 0 +# HELP milvus_querycoord_resource_group_info all resource group detail info in query coord +# TYPE milvus_querycoord_resource_group_info gauge +milvus_querycoord_resource_group_info{node_id="37",rg="__default_resource_group"} 1 +# HELP milvus_querycoord_resource_group_replica_total total replica number of resource group +# TYPE milvus_querycoord_resource_group_replica_total gauge +milvus_querycoord_resource_group_replica_total{rg="__default_resource_group"} 0 +# HELP milvus_querycoord_task_num the number of tasks in QueryCoord's scheduler +# TYPE milvus_querycoord_task_num gauge +milvus_querycoord_task_num{querycoord_task_type="channel_grow"} 0 +milvus_querycoord_task_num{querycoord_task_type="channel_move"} 0 +milvus_querycoord_task_num{querycoord_task_type="channel_reduce"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_grow"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_move"} 0 +milvus_querycoord_task_num{querycoord_task_type="segment_reduce"} 0 +# HELP milvus_querynode_apply_bf_latency apply bf cost in ms +# TYPE milvus_querynode_apply_bf_latency histogram +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="1"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="2"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="4"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="8"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="16"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="32"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="64"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="128"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="256"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="512"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="1024"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="2048"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="4096"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="8192"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="16384"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="32768"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="65536"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="131072"} 1 +milvus_querynode_apply_bf_latency_bucket{function_name="ProcessDelete",node_id="37",le="+Inf"} 1 +milvus_querynode_apply_bf_latency_sum{function_name="ProcessDelete",node_id="37"} 0 +milvus_querynode_apply_bf_latency_count{function_name="ProcessDelete",node_id="37"} 1 +# HELP milvus_querynode_cgo_latency latency of each cgo call +# TYPE milvus_querynode_cgo_latency histogram +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="1"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="2"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="4"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="8"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="16"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="32"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="64"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="128"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="256"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="512"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="1024"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="2048"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="4096"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="8192"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="16384"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="32768"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="65536"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="131072"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37",le="+Inf"} 1 +milvus_querynode_cgo_latency_sum{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_count{cgo_name="AppendIndexV2",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="1"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="2"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="4"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="8"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="16"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="32"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="64"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="128"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="256"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="512"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="1024"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="2048"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="4096"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="8192"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="16384"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="32768"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="65536"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="131072"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="Delete",cgo_type="Sync",node_id="37",le="+Inf"} 1 +milvus_querynode_cgo_latency_sum{cgo_name="Delete",cgo_type="Sync",node_id="37"} 0 +milvus_querynode_cgo_latency_count{cgo_name="Delete",cgo_type="Sync",node_id="37"} 1 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="1"} 3 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="2"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="4"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="8"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="16"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="32"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="64"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="128"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="256"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="512"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="1024"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="2048"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="4096"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="8192"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="16384"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="32768"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="65536"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="131072"} 4 +milvus_querynode_cgo_latency_bucket{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37",le="+Inf"} 4 +milvus_querynode_cgo_latency_sum{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37"} 5 +milvus_querynode_cgo_latency_count{cgo_name="LoadFieldData",cgo_type="Sync",node_id="37"} 4 +# HELP milvus_querynode_collection_num number of collections loaded +# TYPE milvus_querynode_collection_num gauge +milvus_querynode_collection_num{node_id="37"} 0 +# HELP milvus_querynode_consume_bytes_counter +# TYPE milvus_querynode_consume_bytes_counter counter +milvus_querynode_consume_bytes_counter{msg_type="insert",node_id="37"} 241 +# HELP milvus_querynode_disk_cache_evict_bytes number of bytes evicted from disk cache +# TYPE milvus_querynode_disk_cache_evict_bytes counter +milvus_querynode_disk_cache_evict_bytes{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_evict_duration total time cost of evicting segments from disk cache +# TYPE milvus_querynode_disk_cache_evict_duration counter +milvus_querynode_disk_cache_evict_duration{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_evict_global_duration global duration of evicting segments from disk cache +# TYPE milvus_querynode_disk_cache_evict_global_duration histogram +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="100"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="500"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="5000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="10000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="20000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="50000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="100000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="250000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="500000"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="3.6e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="5e+06"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="1e+07"} 0 +milvus_querynode_disk_cache_evict_global_duration_bucket{node_id="37",le="+Inf"} 0 +milvus_querynode_disk_cache_evict_global_duration_sum{node_id="37"} 0 +milvus_querynode_disk_cache_evict_global_duration_count{node_id="37"} 0 +# HELP milvus_querynode_disk_cache_evict_total number of segments evicted from disk cache +# TYPE milvus_querynode_disk_cache_evict_total counter +milvus_querynode_disk_cache_evict_total{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_bytes number of bytes loaded from disk cache +# TYPE milvus_querynode_disk_cache_load_bytes counter +milvus_querynode_disk_cache_load_bytes{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_duration total time cost of loading segments from disk cache +# TYPE milvus_querynode_disk_cache_load_duration counter +milvus_querynode_disk_cache_load_duration{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_cache_load_global_duration global duration of loading segments from disk cache +# TYPE milvus_querynode_disk_cache_load_global_duration histogram +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="100"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="500"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="5000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="10000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="20000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="50000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="100000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="250000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="500000"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="3.6e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="5e+06"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="1e+07"} 0 +milvus_querynode_disk_cache_load_global_duration_bucket{node_id="37",le="+Inf"} 0 +milvus_querynode_disk_cache_load_global_duration_sum{node_id="37"} 0 +milvus_querynode_disk_cache_load_global_duration_count{node_id="37"} 0 +# HELP milvus_querynode_disk_cache_load_total number of segments loaded from disk cache +# TYPE milvus_querynode_disk_cache_load_total counter +milvus_querynode_disk_cache_load_total{db_name="default",node_id="37",rg="__default_resource_group"} 0 +# HELP milvus_querynode_disk_used_size disk used size(MB) +# TYPE milvus_querynode_disk_used_size gauge +milvus_querynode_disk_used_size{node_id="37"} 4.285002708435059 +# HELP milvus_querynode_dml_vchannel_num number of dmlChannels watched +# TYPE milvus_querynode_dml_vchannel_num gauge +milvus_querynode_dml_vchannel_num{node_id="37"} 0 +# HELP milvus_querynode_execute_bytes_counter +# TYPE milvus_querynode_execute_bytes_counter counter +milvus_querynode_execute_bytes_counter{msg_type="query",node_id="37"} 818 +milvus_querynode_execute_bytes_counter{msg_type="search",node_id="37"} 599 +# HELP milvus_querynode_flowgraph_num number of flowgraphs +# TYPE milvus_querynode_flowgraph_num gauge +milvus_querynode_flowgraph_num{node_id="37"} 0 +# HELP milvus_querynode_forward_delete_latency forward delete cost in ms +# TYPE milvus_querynode_forward_delete_latency histogram +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="1"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="2"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="4"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="8"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="16"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="32"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="64"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="128"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="256"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="512"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="1024"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="2048"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="4096"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="8192"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="16384"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="32768"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="65536"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="131072"} 1 +milvus_querynode_forward_delete_latency_bucket{function_name="ProcessDelete",node_id="37",le="+Inf"} 1 +milvus_querynode_forward_delete_latency_sum{function_name="ProcessDelete",node_id="37"} 1 +milvus_querynode_forward_delete_latency_count{function_name="ProcessDelete",node_id="37"} 1 +# HELP milvus_querynode_load_index_latency latency of load per segment's index, in milliseconds +# TYPE milvus_querynode_load_index_latency histogram +milvus_querynode_load_index_latency_bucket{node_id="37",le="1"} 0 +milvus_querynode_load_index_latency_bucket{node_id="37",le="100"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="500"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="5000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="10000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="20000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="50000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="100000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="250000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="500000"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="3.6e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="5e+06"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="1e+07"} 1 +milvus_querynode_load_index_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_load_index_latency_sum{node_id="37"} 2 +milvus_querynode_load_index_latency_count{node_id="37"} 1 +# HELP milvus_querynode_load_segment_concurrency number of concurrent loading segments in QueryNode +# TYPE milvus_querynode_load_segment_concurrency gauge +milvus_querynode_load_segment_concurrency{load_type="LoadSegment",node_id="37"} 0 +# HELP milvus_querynode_load_segment_latency latency of load per segment +# TYPE milvus_querynode_load_segment_latency histogram +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1"} 0 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="100"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="500"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="5000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="10000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="20000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="50000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="100000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="250000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="500000"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="3.6e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="5e+06"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="1e+07"} 1 +milvus_querynode_load_segment_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_load_segment_latency_sum{node_id="37"} 9 +milvus_querynode_load_segment_latency_count{node_id="37"} 1 +# HELP milvus_querynode_process_insert_or_delete_latency process insert or delete cost in ms +# TYPE milvus_querynode_process_insert_or_delete_latency histogram +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="1"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="2"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="4"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="8"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="16"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="32"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="64"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="128"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="256"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="512"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="1024"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="2048"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="4096"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="8192"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="16384"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="32768"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="65536"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="131072"} 1 +milvus_querynode_process_insert_or_delete_latency_bucket{msg_type="delete",node_id="37",le="+Inf"} 1 +milvus_querynode_process_insert_or_delete_latency_sum{msg_type="delete",node_id="37"} 1 +milvus_querynode_process_insert_or_delete_latency_count{msg_type="delete",node_id="37"} 1 +# HELP milvus_querynode_read_task_concurrency number of concurrent executing read tasks in QueryNode +# TYPE milvus_querynode_read_task_concurrency gauge +milvus_querynode_read_task_concurrency{node_id="37"} 0 +# HELP milvus_querynode_read_task_ready_len number of ready read tasks in readyQueue +# TYPE milvus_querynode_read_task_ready_len gauge +milvus_querynode_read_task_ready_len{node_id="37"} 0 +# HELP milvus_querynode_read_task_unsolved_len number of unsolved read tasks in unsolvedQueue +# TYPE milvus_querynode_read_task_unsolved_len gauge +milvus_querynode_read_task_unsolved_len{node_id="37"} 0 +# HELP milvus_querynode_search_group_nq the number of queries of each grouped search task +# TYPE milvus_querynode_search_group_nq histogram +milvus_querynode_search_group_nq_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_group_nq_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_nq_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_nq_sum{node_id="37"} 4 +milvus_querynode_search_group_nq_count{node_id="37"} 2 +# HELP milvus_querynode_search_group_size the number of tasks of each grouped search task +# TYPE milvus_querynode_search_group_size histogram +milvus_querynode_search_group_size_bucket{node_id="37",le="1"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_size_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_size_sum{node_id="37"} 2 +milvus_querynode_search_group_size_count{node_id="37"} 2 +# HELP milvus_querynode_search_group_topk the topK of each grouped search task +# TYPE milvus_querynode_search_group_topk histogram +milvus_querynode_search_group_topk_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_group_topk_bucket{node_id="37",le="2"} 0 +milvus_querynode_search_group_topk_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_group_topk_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_group_topk_sum{node_id="37"} 6 +milvus_querynode_search_group_topk_count{node_id="37"} 2 +# HELP milvus_querynode_search_nq the number of queries of each search task +# TYPE milvus_querynode_search_nq histogram +milvus_querynode_search_nq_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_nq_bucket{node_id="37",le="2"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_nq_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_nq_sum{node_id="37"} 4 +milvus_querynode_search_nq_count{node_id="37"} 2 +# HELP milvus_querynode_search_topk the top of each search task +# TYPE milvus_querynode_search_topk histogram +milvus_querynode_search_topk_bucket{node_id="37",le="1"} 0 +milvus_querynode_search_topk_bucket{node_id="37",le="2"} 0 +milvus_querynode_search_topk_bucket{node_id="37",le="4"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="8"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="16"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="32"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="64"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="128"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="256"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="512"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="1024"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="2048"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="4096"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="8192"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="16384"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="32768"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="65536"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="131072"} 2 +milvus_querynode_search_topk_bucket{node_id="37",le="+Inf"} 2 +milvus_querynode_search_topk_sum{node_id="37"} 6 +milvus_querynode_search_topk_count{node_id="37"} 2 +# HELP milvus_querynode_segment_access_duration total time cost of accessing segments +# TYPE milvus_querynode_segment_access_duration counter +milvus_querynode_segment_access_duration{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 11.942583 +milvus_querynode_segment_access_duration{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 4.55725 +# HELP milvus_querynode_segment_access_global_duration global time cost of accessing segments +# TYPE milvus_querynode_segment_access_global_duration histogram +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="100"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="500"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="5000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="10000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="20000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="50000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="100000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="250000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="500000"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="3.6e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="5e+06"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="1e+07"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_segment_access_global_duration_sum{node_id="37",query_type="query"} 11.942583 +milvus_querynode_segment_access_global_duration_count{node_id="37",query_type="query"} 5 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="100"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="500"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="5000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="10000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="20000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="50000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="100000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="250000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="500000"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="3.6e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="5e+06"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="1e+07"} 2 +milvus_querynode_segment_access_global_duration_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_segment_access_global_duration_sum{node_id="37",query_type="search"} 4.55725 +milvus_querynode_segment_access_global_duration_count{node_id="37",query_type="search"} 2 +# HELP milvus_querynode_segment_access_total number of segments accessed +# TYPE milvus_querynode_segment_access_total counter +milvus_querynode_segment_access_total{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_segment_access_total{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 2 +# HELP milvus_querynode_segment_access_wait_cache_duration total time cost of waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_duration counter +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_duration{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0 +# HELP milvus_querynode_segment_access_wait_cache_global_duration global time cost of waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_global_duration histogram +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="query",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="37",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="37",query_type="query"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="100"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="500"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="5000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="10000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="20000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="50000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="100000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="250000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="500000"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="3.6e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="5e+06"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="1e+07"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_bucket{node_id="37",query_type="search",le="+Inf"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_sum{node_id="37",query_type="search"} 0 +milvus_querynode_segment_access_wait_cache_global_duration_count{node_id="37",query_type="search"} 0 +# HELP milvus_querynode_segment_access_wait_cache_total number of segments waiting for loading access +# TYPE milvus_querynode_segment_access_wait_cache_total counter +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0 +milvus_querynode_segment_access_wait_cache_total{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0 +# HELP milvus_querynode_segment_latency_per_vector one vector's search latency per segment +# TYPE milvus_querynode_segment_latency_per_vector histogram +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="64"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="128"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="256"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="512"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1024"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2048"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4096"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8192"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16384"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32768"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="65536"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="131072"} 2 +milvus_querynode_segment_latency_per_vector_bucket{node_id="37",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_segment_latency_per_vector_sum{node_id="37",query_type="search",segment_state="Sealed"} 1.5 +milvus_querynode_segment_latency_per_vector_count{node_id="37",query_type="search",segment_state="Sealed"} 2 +# HELP milvus_querynode_sq_core_latency latency of search or query latency in segcore +# TYPE milvus_querynode_sq_core_latency histogram +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="1"} 4 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="2"} 4 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="4"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="8"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="16"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="32"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="64"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="128"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="256"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_core_latency_sum{node_id="37",query_type="query"} 7 +milvus_querynode_sq_core_latency_count{node_id="37",query_type="query"} 5 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="1"} 1 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="2"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="4"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="8"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="16"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="32"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="64"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="128"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="256"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_querynode_sq_core_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_core_latency_sum{node_id="37",query_type="search"} 3 +milvus_querynode_sq_core_latency_count{node_id="37",query_type="search"} 2 +# HELP milvus_querynode_sq_queue_latency latency of search or query in queue +# TYPE milvus_querynode_sq_queue_latency histogram +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="1"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="2"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="4"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="8"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="16"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="32"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="64"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="128"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="256"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="512"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="1024"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="2048"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="4096"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="8192"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="16384"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="32768"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="65536"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="131072"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="query",rg="__default_resource_group",le="+Inf"} 5 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 0.39716599999999996 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="37",query_type="query",rg="__default_resource_group"} 5 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="1"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="2"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="4"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="8"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="16"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="32"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="64"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="128"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="256"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="512"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="1024"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="2048"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="4096"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="8192"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="16384"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="32768"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="65536"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="131072"} 2 +milvus_querynode_sq_queue_latency_bucket{db_name="default",node_id="37",query_type="search",rg="__default_resource_group",le="+Inf"} 2 +milvus_querynode_sq_queue_latency_sum{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 0.12625 +milvus_querynode_sq_queue_latency_count{db_name="default",node_id="37",query_type="search",rg="__default_resource_group"} 2 +# HELP milvus_querynode_sq_queue_user_latency latency per user of search or query in queue +# TYPE milvus_querynode_sq_queue_user_latency histogram +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="1"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="2"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="4"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="8"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="16"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="32"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="64"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="128"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="256"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="512"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="1024"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="2048"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="4096"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="8192"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="16384"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="32768"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="65536"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="131072"} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="query",username="",le="+Inf"} 5 +milvus_querynode_sq_queue_user_latency_sum{node_id="37",query_type="query",username=""} 0.39716599999999996 +milvus_querynode_sq_queue_user_latency_count{node_id="37",query_type="query",username=""} 5 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="1"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="2"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="4"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="8"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="16"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="32"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="64"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="128"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="256"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="512"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="1024"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="2048"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="4096"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="8192"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="16384"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="32768"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="65536"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="131072"} 2 +milvus_querynode_sq_queue_user_latency_bucket{node_id="37",query_type="search",username="",le="+Inf"} 2 +milvus_querynode_sq_queue_user_latency_sum{node_id="37",query_type="search",username=""} 0.12625 +milvus_querynode_sq_queue_user_latency_count{node_id="37",query_type="search",username=""} 2 +# HELP milvus_querynode_sq_reduce_latency latency of reduce search or query result +# TYPE milvus_querynode_sq_reduce_latency histogram +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="2"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="4"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="8"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="16"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="32"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="64"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="128"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="256"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="512"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="1024"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="2048"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="4096"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="8192"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="16384"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="32768"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="65536"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="131072"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 1 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="query",reduce_level="segments",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="2"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="4"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="8"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="16"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="32"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="64"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="128"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="256"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="512"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="1024"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="2048"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="4096"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="8192"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="16384"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="32768"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="65536"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="131072"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 5 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="query",reduce_level="shards",reduce_type="batch_reduce"} 5 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="2"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="4"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="8"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="16"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="32"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="64"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="128"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="256"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="512"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="1024"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="2048"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="4096"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="8192"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="16384"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="32768"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="65536"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="131072"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="search",reduce_level="segments",reduce_type="batch_reduce"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="2"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="4"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="8"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="16"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="32"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="64"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="128"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="256"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="512"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="1024"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="2048"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="4096"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="8192"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="16384"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="32768"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="65536"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="131072"} 2 +milvus_querynode_sq_reduce_latency_bucket{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce",le="+Inf"} 2 +milvus_querynode_sq_reduce_latency_sum{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 0 +milvus_querynode_sq_reduce_latency_count{node_id="37",query_type="search",reduce_level="shards",reduce_type="batch_reduce"} 2 +# HELP milvus_querynode_sq_req_latency latency of Search or query requests +# TYPE milvus_querynode_sq_req_latency histogram +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="1"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="2"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="4"} 4 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="8"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="16"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="32"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="64"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="128"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="256"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="512"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="1024"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="2048"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="4096"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="8192"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="16384"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="32768"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="65536"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="131072"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="FromLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="query",scope="FromLeader"} 14 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="query",scope="FromLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="2"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="4"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="8"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="16"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="32"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="64"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="128"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="256"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="512"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="1024"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="2048"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="4096"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="8192"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="16384"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="32768"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="65536"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="131072"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="query",scope="OnLeader",le="+Inf"} 5 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="query",scope="OnLeader"} 1469 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="query",scope="OnLeader"} 5 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="2"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="4"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="8"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="16"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="32"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="64"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="128"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="256"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="512"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="1024"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="2048"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="4096"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="8192"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="16384"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="32768"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="65536"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="131072"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="FromLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="search",scope="FromLeader"} 6 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="search",scope="FromLeader"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="1"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="2"} 0 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="4"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="8"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="16"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="32"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="64"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="128"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="256"} 1 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="512"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="1024"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="2048"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="4096"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="8192"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="16384"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="32768"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="65536"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="131072"} 2 +milvus_querynode_sq_req_latency_bucket{node_id="37",query_type="search",scope="OnLeader",le="+Inf"} 2 +milvus_querynode_sq_req_latency_sum{node_id="37",query_type="search",scope="OnLeader"} 405 +milvus_querynode_sq_req_latency_count{node_id="37",query_type="search",scope="OnLeader"} 2 +# HELP milvus_querynode_sq_segment_latency latency of search or query per segment +# TYPE milvus_querynode_sq_segment_latency histogram +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="1"} 3 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="2"} 4 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="4"} 4 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="8"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="16"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="32"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="64"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="128"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="256"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="512"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="1024"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="2048"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="4096"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="8192"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="16384"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="32768"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="65536"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="131072"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="query",segment_state="Sealed",le="+Inf"} 5 +milvus_querynode_sq_segment_latency_sum{node_id="37",query_type="query",segment_state="Sealed"} 10 +milvus_querynode_sq_segment_latency_count{node_id="37",query_type="query",segment_state="Sealed"} 5 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1"} 1 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="64"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="128"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="256"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="512"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="1024"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="2048"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="4096"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="8192"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="16384"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="32768"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="65536"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="131072"} 2 +milvus_querynode_sq_segment_latency_bucket{node_id="37",query_type="search",segment_state="Sealed",le="+Inf"} 2 +milvus_querynode_sq_segment_latency_sum{node_id="37",query_type="search",segment_state="Sealed"} 3 +milvus_querynode_sq_segment_latency_count{node_id="37",query_type="search",segment_state="Sealed"} 2 +# HELP milvus_querynode_sq_wait_tsafe_latency latency of search or query to wait for tsafe +# TYPE milvus_querynode_sq_wait_tsafe_latency histogram +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="1"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="2"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="4"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="8"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="16"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="32"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="64"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="128"} 0 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="256"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="512"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="1024"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="2048"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="4096"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="8192"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="16384"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="32768"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="65536"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="131072"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="query",le="+Inf"} 5 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="37",query_type="query"} 1449 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="37",query_type="query"} 5 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="1"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="2"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="4"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="8"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="16"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="32"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="64"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="128"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="256"} 1 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="512"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="1024"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="2048"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="4096"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="8192"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="16384"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="32768"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="65536"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="131072"} 2 +milvus_querynode_sq_wait_tsafe_latency_bucket{node_id="37",query_type="search",le="+Inf"} 2 +milvus_querynode_sq_wait_tsafe_latency_sum{node_id="37",query_type="search"} 398 +milvus_querynode_sq_wait_tsafe_latency_count{node_id="37",query_type="search"} 2 +# HELP milvus_querynode_wait_processing_msg_count count of wait processing msg +# TYPE milvus_querynode_wait_processing_msg_count gauge +milvus_querynode_wait_processing_msg_count{msg_type="delete",node_id="37"} 0 +milvus_querynode_wait_processing_msg_count{msg_type="insert",node_id="37"} 0 +# HELP milvus_querynode_watch_dml_channel_latency latency of watch dml channel +# TYPE milvus_querynode_watch_dml_channel_latency histogram +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="1"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="2"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="4"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="8"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="16"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="32"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="64"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="128"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="256"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="512"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="1024"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="2048"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="4096"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="8192"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="16384"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="32768"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="65536"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="131072"} 1 +milvus_querynode_watch_dml_channel_latency_bucket{node_id="37",le="+Inf"} 1 +milvus_querynode_watch_dml_channel_latency_sum{node_id="37"} 0 +milvus_querynode_watch_dml_channel_latency_count{node_id="37"} 1 +# HELP milvus_rootcoord_collection_num number of collections +# TYPE milvus_rootcoord_collection_num gauge +milvus_rootcoord_collection_num{db_name="default"} 0 +# HELP milvus_rootcoord_credential_num number of credentials +# TYPE milvus_rootcoord_credential_num gauge +milvus_rootcoord_credential_num 0 +# HELP milvus_rootcoord_ddl_req_count count of DDL operations +# TYPE milvus_rootcoord_ddl_req_count counter +milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="CreateCollection",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="fail"} 3 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="success"} 12 +milvus_rootcoord_ddl_req_count{function_name="DescribeCollection",status="total"} 15 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="success"} 3 +milvus_rootcoord_ddl_req_count{function_name="DescribeDatabase",status="total"} 3 +milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="DropCollection",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ListDatabases",status="success"} 2 +milvus_rootcoord_ddl_req_count{function_name="ListDatabases",status="total"} 2 +milvus_rootcoord_ddl_req_count{function_name="PolicyList",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="PolicyList",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowCollections",status="success"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowCollections",status="total"} 1 +milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="success"} 4 +milvus_rootcoord_ddl_req_count{function_name="ShowPartitions",status="total"} 4 +# HELP milvus_rootcoord_ddl_req_latency latency of each DDL operations +# TYPE milvus_rootcoord_ddl_req_latency histogram +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="2"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="131072"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="CreateCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="CreateCollection"} 4 +milvus_rootcoord_ddl_req_latency_count{function_name="CreateCollection"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="2"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="4"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="8"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="16"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="32"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="64"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="128"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="256"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="512"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="1024"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="2048"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="4096"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="8192"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="16384"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="32768"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="65536"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="131072"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeCollection",le="+Inf"} 12 +milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeCollection"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeCollection"} 12 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="2"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="4"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="8"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="16"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="32"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="64"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="128"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="256"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="512"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="1024"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="2048"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="4096"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="8192"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="16384"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="32768"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="65536"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="131072"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DescribeDatabase",le="+Inf"} 3 +milvus_rootcoord_ddl_req_latency_sum{function_name="DescribeDatabase"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="DescribeDatabase"} 3 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="2"} 0 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="131072"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="DropCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="DropCollection"} 4 +milvus_rootcoord_ddl_req_latency_count{function_name="DropCollection"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="2"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="4"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="8"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="16"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="32"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="64"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="128"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="256"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="512"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="1024"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="2048"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="4096"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="8192"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="16384"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="32768"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="65536"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="131072"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ListDatabases",le="+Inf"} 2 +milvus_rootcoord_ddl_req_latency_sum{function_name="ListDatabases"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ListDatabases"} 2 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="2"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="131072"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="PolicyList",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="PolicyList"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="PolicyList"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="2"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="4"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="8"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="16"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="32"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="64"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="128"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="256"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="512"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="1024"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="2048"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="4096"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="8192"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="16384"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="32768"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="65536"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="131072"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowCollections",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_sum{function_name="ShowCollections"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ShowCollections"} 1 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="2"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="4"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="8"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="16"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="32"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="64"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="128"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="256"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="512"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="1024"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="2048"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="4096"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="8192"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="16384"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="32768"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="65536"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="131072"} 4 +milvus_rootcoord_ddl_req_latency_bucket{function_name="ShowPartitions",le="+Inf"} 4 +milvus_rootcoord_ddl_req_latency_sum{function_name="ShowPartitions"} 0 +milvus_rootcoord_ddl_req_latency_count{function_name="ShowPartitions"} 4 +# HELP milvus_rootcoord_ddl_req_latency_in_queue latency of each DDL operations in queue +# TYPE milvus_rootcoord_ddl_req_latency_in_queue histogram +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="CreateCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="CreateCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="CreateCollection"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.005"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.01"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.025"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.05"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.1"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.25"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="0.5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="1"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="2.5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="5"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="10"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DescribeCollection",le="+Inf"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="DescribeCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DescribeCollection"} 12 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="DropCollection",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="DropCollection"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="DropCollection"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.005"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.01"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.025"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.05"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.25"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="0.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="1"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="2.5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="5"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="10"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowCollections",le="+Inf"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="ShowCollections"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="ShowCollections"} 1 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.005"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.01"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.025"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.05"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.1"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.25"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="0.5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="1"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="2.5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="5"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="10"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_bucket{function_name="ShowPartitions",le="+Inf"} 4 +milvus_rootcoord_ddl_req_latency_in_queue_sum{function_name="ShowPartitions"} 0 +milvus_rootcoord_ddl_req_latency_in_queue_count{function_name="ShowPartitions"} 4 +# HELP milvus_rootcoord_dml_channel_num number of DML channels +# TYPE milvus_rootcoord_dml_channel_num gauge +milvus_rootcoord_dml_channel_num 16 +# HELP milvus_rootcoord_entity_num number of entities, clustered by collection and their status(loaded/total) +# TYPE milvus_rootcoord_entity_num gauge +milvus_rootcoord_entity_num{collection_name="hello_milvus",status="loaded"} 3001 +milvus_rootcoord_entity_num{collection_name="hello_milvus",status="total"} 3001 +# HELP milvus_rootcoord_force_deny_writing_counter The number of times milvus turns into force-deny-writing states +# TYPE milvus_rootcoord_force_deny_writing_counter counter +milvus_rootcoord_force_deny_writing_counter 0 +# HELP milvus_rootcoord_id_alloc_count count of ID allocated +# TYPE milvus_rootcoord_id_alloc_count counter +milvus_rootcoord_id_alloc_count 400508 +# HELP milvus_rootcoord_indexed_entity_num indexed number of entities, clustered by collection, index name and whether it's a vector index +# TYPE milvus_rootcoord_indexed_entity_num gauge +milvus_rootcoord_indexed_entity_num{collection_name="hello_milvus",index_name="embeddings",is_vector_index="true"} 3001 +# HELP milvus_rootcoord_msgstream_obj_num number of message streams +# TYPE milvus_rootcoord_msgstream_obj_num gauge +milvus_rootcoord_msgstream_obj_num 16 +# HELP milvus_rootcoord_num_of_roles The number of roles +# TYPE milvus_rootcoord_num_of_roles gauge +milvus_rootcoord_num_of_roles 0 +# HELP milvus_rootcoord_partition_num number of partitions +# TYPE milvus_rootcoord_partition_num gauge +milvus_rootcoord_partition_num 0 +# HELP milvus_rootcoord_produce_tt_lag_ms now time minus tt per physical channel +# TYPE milvus_rootcoord_produce_tt_lag_ms gauge +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_0"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_1"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_10"} 137 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_2"} 87 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_3"} 94 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_4"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_5"} 96 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_6"} 95 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_7"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_8"} 97 +milvus_rootcoord_produce_tt_lag_ms{channel_name="by-dev-rootcoord-dml_9"} 97 +# HELP milvus_rootcoord_proxy_num number of proxy nodes managered by rootcoord +# TYPE milvus_rootcoord_proxy_num gauge +milvus_rootcoord_proxy_num 1 +# HELP milvus_rootcoord_qn_mem_high_water_level querynode memory high water level +# TYPE milvus_rootcoord_qn_mem_high_water_level gauge +milvus_rootcoord_qn_mem_high_water_level 0 +# HELP milvus_rootcoord_sync_timetick_latency latency of synchronizing timetick message +# TYPE milvus_rootcoord_sync_timetick_latency histogram +milvus_rootcoord_sync_timetick_latency_bucket{le="1"} 42 +milvus_rootcoord_sync_timetick_latency_bucket{le="2"} 65 +milvus_rootcoord_sync_timetick_latency_bucket{le="4"} 181 +milvus_rootcoord_sync_timetick_latency_bucket{le="8"} 310 +milvus_rootcoord_sync_timetick_latency_bucket{le="16"} 349 +milvus_rootcoord_sync_timetick_latency_bucket{le="32"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="64"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="128"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="256"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="512"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="1024"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="2048"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="4096"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="8192"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="16384"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="32768"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="65536"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="131072"} 353 +milvus_rootcoord_sync_timetick_latency_bucket{le="+Inf"} 353 +milvus_rootcoord_sync_timetick_latency_sum 1756 +milvus_rootcoord_sync_timetick_latency_count 353 +# HELP milvus_rootcoord_timestamp lateste timestamp allocated in memory +# TYPE milvus_rootcoord_timestamp gauge +milvus_rootcoord_timestamp 4.548092784887726e+17 +# HELP milvus_rootcoord_timestamp_saved timestamp saved in meta storage +# TYPE milvus_rootcoord_timestamp_saved gauge +milvus_rootcoord_timestamp_saved 1.734959711e+09 +# HELP milvus_runtime_info Runtime information of milvus +# TYPE milvus_runtime_info gauge +milvus_runtime_info{meta="etcd",mq="rocksmq"} 1 +# HELP milvus_thread_num the actual thread number of milvus process +# TYPE milvus_thread_num gauge +milvus_thread_num 73 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 8.06 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 71 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 5.58592e+08 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.73495960802e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 8.522620928e+09 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes 1.8446744073709552e+19 +# HELP ann_iterator_init_latency ann iterator init latency (ms) +# TYPE ann_iterator_init_latency histogram +ann_iterator_init_latency_bucket{module="cardinal",le="1"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="2"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="4"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="8"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="16"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="32"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="64"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="128"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="256"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="512"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="1024"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="2048"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="4096"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="8192"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="16384"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="32768"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="65536"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +ann_iterator_init_latency_bucket{module="cardinal",le="+Inf"} 0 +ann_iterator_init_latency_sum{module="cardinal"} 0 +ann_iterator_init_latency_count{module="cardinal"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="2"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="4"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="8"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="16"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="32"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="64"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="128"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="256"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="512"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1024"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="2048"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="4096"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="8192"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="16384"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="32768"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="65536"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="1.048576e+06"} 0 +ann_iterator_init_latency_bucket{module="knowhere",le="+Inf"} 0 +ann_iterator_init_latency_sum{module="knowhere"} 0 +ann_iterator_init_latency_count{module="knowhere"} 0 +# HELP bf_search_cnt number of bf search per request +# TYPE bf_search_cnt histogram +bf_search_cnt_bucket{module="cardinal",le="1"} 0 +bf_search_cnt_bucket{module="cardinal",le="2"} 0 +bf_search_cnt_bucket{module="cardinal",le="4"} 0 +bf_search_cnt_bucket{module="cardinal",le="8"} 0 +bf_search_cnt_bucket{module="cardinal",le="16"} 0 +bf_search_cnt_bucket{module="cardinal",le="32"} 0 +bf_search_cnt_bucket{module="cardinal",le="64"} 0 +bf_search_cnt_bucket{module="cardinal",le="128"} 0 +bf_search_cnt_bucket{module="cardinal",le="256"} 0 +bf_search_cnt_bucket{module="cardinal",le="512"} 0 +bf_search_cnt_bucket{module="cardinal",le="1024"} 0 +bf_search_cnt_bucket{module="cardinal",le="2048"} 0 +bf_search_cnt_bucket{module="cardinal",le="4096"} 0 +bf_search_cnt_bucket{module="cardinal",le="8192"} 0 +bf_search_cnt_bucket{module="cardinal",le="16384"} 0 +bf_search_cnt_bucket{module="cardinal",le="32768"} 0 +bf_search_cnt_bucket{module="cardinal",le="65536"} 0 +bf_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +bf_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +bf_search_cnt_sum{module="cardinal"} 0 +bf_search_cnt_count{module="cardinal"} 0 +# HELP bitset_ratio bitset ratio +# TYPE bitset_ratio histogram +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.05"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.1"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.15"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.2"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.25"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.3"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.35"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.4"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.45"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.5"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.55"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.6"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.65"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.7"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.75"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.8"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.85"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.9"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="0.95"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="1"} 0 +bitset_ratio_bucket{module="PROMETHEUS_LABEL_CARDINAL",le="+Inf"} 0 +bitset_ratio_sum{module="PROMETHEUS_LABEL_CARDINAL"} 0 +bitset_ratio_count{module="PROMETHEUS_LABEL_CARDINAL"} 0 +# HELP build_latency index build latency (s) +# TYPE build_latency histogram +build_latency_bucket{module="cardinal",le="1"} 0 +build_latency_bucket{module="cardinal",le="2"} 0 +build_latency_bucket{module="cardinal",le="4"} 0 +build_latency_bucket{module="cardinal",le="8"} 0 +build_latency_bucket{module="cardinal",le="16"} 0 +build_latency_bucket{module="cardinal",le="32"} 0 +build_latency_bucket{module="cardinal",le="64"} 0 +build_latency_bucket{module="cardinal",le="128"} 0 +build_latency_bucket{module="cardinal",le="256"} 0 +build_latency_bucket{module="cardinal",le="512"} 0 +build_latency_bucket{module="cardinal",le="1024"} 0 +build_latency_bucket{module="cardinal",le="2048"} 0 +build_latency_bucket{module="cardinal",le="4096"} 0 +build_latency_bucket{module="cardinal",le="8192"} 0 +build_latency_bucket{module="cardinal",le="16384"} 0 +build_latency_bucket{module="cardinal",le="32768"} 0 +build_latency_bucket{module="cardinal",le="65536"} 0 +build_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +build_latency_bucket{module="cardinal",le="+Inf"} 0 +build_latency_sum{module="cardinal"} 0 +build_latency_count{module="cardinal"} 0 +build_latency_bucket{module="knowhere",le="1"} 1 +build_latency_bucket{module="knowhere",le="2"} 1 +build_latency_bucket{module="knowhere",le="4"} 1 +build_latency_bucket{module="knowhere",le="8"} 1 +build_latency_bucket{module="knowhere",le="16"} 1 +build_latency_bucket{module="knowhere",le="32"} 1 +build_latency_bucket{module="knowhere",le="64"} 1 +build_latency_bucket{module="knowhere",le="128"} 1 +build_latency_bucket{module="knowhere",le="256"} 1 +build_latency_bucket{module="knowhere",le="512"} 1 +build_latency_bucket{module="knowhere",le="1024"} 1 +build_latency_bucket{module="knowhere",le="2048"} 1 +build_latency_bucket{module="knowhere",le="4096"} 1 +build_latency_bucket{module="knowhere",le="8192"} 1 +build_latency_bucket{module="knowhere",le="16384"} 1 +build_latency_bucket{module="knowhere",le="32768"} 1 +build_latency_bucket{module="knowhere",le="65536"} 1 +build_latency_bucket{module="knowhere",le="1.048576e+06"} 1 +build_latency_bucket{module="knowhere",le="+Inf"} 1 +build_latency_sum{module="knowhere"} 0.015760917 +build_latency_count{module="knowhere"} 1 +# HELP cache_hit_cnt cache hit cnt per request +# TYPE cache_hit_cnt histogram +cache_hit_cnt_bucket{module="cardinal",le="1"} 0 +cache_hit_cnt_bucket{module="cardinal",le="2"} 0 +cache_hit_cnt_bucket{module="cardinal",le="4"} 0 +cache_hit_cnt_bucket{module="cardinal",le="8"} 0 +cache_hit_cnt_bucket{module="cardinal",le="16"} 0 +cache_hit_cnt_bucket{module="cardinal",le="32"} 0 +cache_hit_cnt_bucket{module="cardinal",le="64"} 0 +cache_hit_cnt_bucket{module="cardinal",le="128"} 0 +cache_hit_cnt_bucket{module="cardinal",le="256"} 0 +cache_hit_cnt_bucket{module="cardinal",le="512"} 0 +cache_hit_cnt_bucket{module="cardinal",le="1024"} 0 +cache_hit_cnt_bucket{module="cardinal",le="2048"} 0 +cache_hit_cnt_bucket{module="cardinal",le="4096"} 0 +cache_hit_cnt_bucket{module="cardinal",le="8192"} 0 +cache_hit_cnt_bucket{module="cardinal",le="16384"} 0 +cache_hit_cnt_bucket{module="cardinal",le="32768"} 0 +cache_hit_cnt_bucket{module="cardinal",le="65536"} 0 +cache_hit_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +cache_hit_cnt_bucket{module="cardinal",le="+Inf"} 0 +cache_hit_cnt_sum{module="cardinal"} 0 +cache_hit_cnt_count{module="cardinal"} 0 +# HELP diskann_bitset_ratio DISKANN bitset ratio for search and range search +# TYPE diskann_bitset_ratio histogram +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.05"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.1"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.15"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.2"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.25"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.3"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.35"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.4"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.45"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.5"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.55"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.6"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.65"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.7"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.75"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.8"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.85"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.9"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.95"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="1"} 0 +diskann_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +diskann_bitset_ratio_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +diskann_bitset_ratio_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP diskann_range_search_iters DISKANN range search iterations +# TYPE diskann_range_search_iters histogram +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="2"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="4"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="6"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="8"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="10"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="12"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="14"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="16"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="18"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="20"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="22"} 0 +diskann_range_search_iters_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +diskann_range_search_iters_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +diskann_range_search_iters_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP diskann_search_hops DISKANN search hops +# TYPE diskann_search_hops histogram +diskann_search_hops_bucket{module="knowhere",le="1"} 0 +diskann_search_hops_bucket{module="knowhere",le="2"} 0 +diskann_search_hops_bucket{module="knowhere",le="4"} 0 +diskann_search_hops_bucket{module="knowhere",le="8"} 0 +diskann_search_hops_bucket{module="knowhere",le="16"} 0 +diskann_search_hops_bucket{module="knowhere",le="32"} 0 +diskann_search_hops_bucket{module="knowhere",le="64"} 0 +diskann_search_hops_bucket{module="knowhere",le="128"} 0 +diskann_search_hops_bucket{module="knowhere",le="256"} 0 +diskann_search_hops_bucket{module="knowhere",le="512"} 0 +diskann_search_hops_bucket{module="knowhere",le="1024"} 0 +diskann_search_hops_bucket{module="knowhere",le="2048"} 0 +diskann_search_hops_bucket{module="knowhere",le="4096"} 0 +diskann_search_hops_bucket{module="knowhere",le="8192"} 0 +diskann_search_hops_bucket{module="knowhere",le="16384"} 0 +diskann_search_hops_bucket{module="knowhere",le="32768"} 0 +diskann_search_hops_bucket{module="knowhere",le="65536"} 0 +diskann_search_hops_bucket{module="knowhere",le="1.048576e+06"} 0 +diskann_search_hops_bucket{module="knowhere",le="+Inf"} 0 +diskann_search_hops_sum{module="knowhere"} 0 +diskann_search_hops_count{module="knowhere"} 0 +# HELP exec_latency execute latency per request +# TYPE exec_latency histogram +exec_latency_bucket{module="cardinal",le="1"} 0 +exec_latency_bucket{module="cardinal",le="2"} 0 +exec_latency_bucket{module="cardinal",le="4"} 0 +exec_latency_bucket{module="cardinal",le="8"} 0 +exec_latency_bucket{module="cardinal",le="16"} 0 +exec_latency_bucket{module="cardinal",le="32"} 0 +exec_latency_bucket{module="cardinal",le="64"} 0 +exec_latency_bucket{module="cardinal",le="128"} 0 +exec_latency_bucket{module="cardinal",le="256"} 0 +exec_latency_bucket{module="cardinal",le="512"} 0 +exec_latency_bucket{module="cardinal",le="1024"} 0 +exec_latency_bucket{module="cardinal",le="2048"} 0 +exec_latency_bucket{module="cardinal",le="4096"} 0 +exec_latency_bucket{module="cardinal",le="8192"} 0 +exec_latency_bucket{module="cardinal",le="16384"} 0 +exec_latency_bucket{module="cardinal",le="32768"} 0 +exec_latency_bucket{module="cardinal",le="65536"} 0 +exec_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +exec_latency_bucket{module="cardinal",le="+Inf"} 0 +exec_latency_sum{module="cardinal"} 0 +exec_latency_count{module="cardinal"} 0 +# HELP filter_connectivity_ratio avg connectivity ratio set under filtering per request +# TYPE filter_connectivity_ratio histogram +filter_connectivity_ratio_bucket{module="cardinal",le="1"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="2"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="4"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="8"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="16"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="32"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="64"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="128"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="256"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="512"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="1024"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="2048"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="4096"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="8192"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="16384"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="32768"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="65536"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_connectivity_ratio_bucket{module="cardinal",le="+Inf"} 0 +filter_connectivity_ratio_sum{module="cardinal"} 0 +filter_connectivity_ratio_count{module="cardinal"} 0 +# HELP filter_mv_activated_fields_cnt avg mv activated fields per request +# TYPE filter_mv_activated_fields_cnt histogram +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_activated_fields_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_activated_fields_cnt_sum{module="cardinal"} 0 +filter_mv_activated_fields_cnt_count{module="cardinal"} 0 +# HELP filter_mv_change_base_cnt mv change base cnt per request +# TYPE filter_mv_change_base_cnt histogram +filter_mv_change_base_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_change_base_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_change_base_cnt_sum{module="cardinal"} 0 +filter_mv_change_base_cnt_count{module="cardinal"} 0 +# HELP filter_mv_only_cnt mv only cnt per request +# TYPE filter_mv_only_cnt histogram +filter_mv_only_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_only_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_only_cnt_sum{module="cardinal"} 0 +filter_mv_only_cnt_count{module="cardinal"} 0 +# HELP filter_mv_supplement_ep_bool_cnt mv supplement ep from bitset boolean cnt per request +# TYPE filter_mv_supplement_ep_bool_cnt histogram +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="2"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="4"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="8"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="16"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="32"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="64"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="128"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="256"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="512"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1024"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="2048"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="4096"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="8192"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="16384"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="32768"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="65536"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +filter_mv_supplement_ep_bool_cnt_bucket{module="cardinal",le="+Inf"} 0 +filter_mv_supplement_ep_bool_cnt_sum{module="cardinal"} 0 +filter_mv_supplement_ep_bool_cnt_count{module="cardinal"} 0 +# HELP graph_search_cnt number of graph search per request +# TYPE graph_search_cnt histogram +graph_search_cnt_bucket{module="cardinal",le="1"} 0 +graph_search_cnt_bucket{module="cardinal",le="2"} 0 +graph_search_cnt_bucket{module="cardinal",le="4"} 0 +graph_search_cnt_bucket{module="cardinal",le="8"} 0 +graph_search_cnt_bucket{module="cardinal",le="16"} 0 +graph_search_cnt_bucket{module="cardinal",le="32"} 0 +graph_search_cnt_bucket{module="cardinal",le="64"} 0 +graph_search_cnt_bucket{module="cardinal",le="128"} 0 +graph_search_cnt_bucket{module="cardinal",le="256"} 0 +graph_search_cnt_bucket{module="cardinal",le="512"} 0 +graph_search_cnt_bucket{module="cardinal",le="1024"} 0 +graph_search_cnt_bucket{module="cardinal",le="2048"} 0 +graph_search_cnt_bucket{module="cardinal",le="4096"} 0 +graph_search_cnt_bucket{module="cardinal",le="8192"} 0 +graph_search_cnt_bucket{module="cardinal",le="16384"} 0 +graph_search_cnt_bucket{module="cardinal",le="32768"} 0 +graph_search_cnt_bucket{module="cardinal",le="65536"} 0 +graph_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +graph_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +graph_search_cnt_sum{module="cardinal"} 0 +graph_search_cnt_count{module="cardinal"} 0 +# HELP hnsw_bitset_ratio HNSW bitset ratio for search and range search +# TYPE hnsw_bitset_ratio histogram +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.05"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.1"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.15"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.2"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.25"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.3"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.35"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.4"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.45"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.5"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.55"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.6"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.65"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.7"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.75"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.8"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.85"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.9"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="0.95"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="1"} 0 +hnsw_bitset_ratio_bucket{module="PROMETHEUS_LABEL_KNOWHERE",le="+Inf"} 0 +hnsw_bitset_ratio_sum{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +hnsw_bitset_ratio_count{module="PROMETHEUS_LABEL_KNOWHERE"} 0 +# HELP hnsw_search_hops HNSW search hops in layer 0 +# TYPE hnsw_search_hops histogram +hnsw_search_hops_bucket{module="knowhere",le="1"} 0 +hnsw_search_hops_bucket{module="knowhere",le="2"} 0 +hnsw_search_hops_bucket{module="knowhere",le="4"} 0 +hnsw_search_hops_bucket{module="knowhere",le="8"} 0 +hnsw_search_hops_bucket{module="knowhere",le="16"} 0 +hnsw_search_hops_bucket{module="knowhere",le="32"} 0 +hnsw_search_hops_bucket{module="knowhere",le="64"} 0 +hnsw_search_hops_bucket{module="knowhere",le="128"} 0 +hnsw_search_hops_bucket{module="knowhere",le="256"} 0 +hnsw_search_hops_bucket{module="knowhere",le="512"} 0 +hnsw_search_hops_bucket{module="knowhere",le="1024"} 0 +hnsw_search_hops_bucket{module="knowhere",le="2048"} 0 +hnsw_search_hops_bucket{module="knowhere",le="4096"} 0 +hnsw_search_hops_bucket{module="knowhere",le="8192"} 0 +hnsw_search_hops_bucket{module="knowhere",le="16384"} 0 +hnsw_search_hops_bucket{module="knowhere",le="32768"} 0 +hnsw_search_hops_bucket{module="knowhere",le="65536"} 0 +hnsw_search_hops_bucket{module="knowhere",le="1.048576e+06"} 0 +hnsw_search_hops_bucket{module="knowhere",le="+Inf"} 0 +hnsw_search_hops_sum{module="knowhere"} 0 +hnsw_search_hops_count{module="knowhere"} 0 +# HELP internal_core_search_latency [cpp]latency(us) of search on segment +# TYPE internal_core_search_latency histogram +internal_core_search_latency_bucket{type="groupby_latency",le="1"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="2"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="4"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="8"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="16"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="32"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="64"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="128"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="256"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="512"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="1024"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="2048"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="4096"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="8192"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="16384"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="32768"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="65536"} 0 +internal_core_search_latency_bucket{type="groupby_latency",le="+Inf"} 0 +internal_core_search_latency_sum{type="groupby_latency"} 0 +internal_core_search_latency_count{type="groupby_latency"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="1"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="2"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="4"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="8"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="16"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="32"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="64"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="128"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="256"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="512"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="1024"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="2048"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="4096"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="8192"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="16384"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="32768"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="65536"} 0 +internal_core_search_latency_bucket{type="iterative_filter_latency",le="+Inf"} 0 +internal_core_search_latency_sum{type="iterative_filter_latency"} 0 +internal_core_search_latency_count{type="iterative_filter_latency"} 0 +internal_core_search_latency_bucket{type="scalar_latency",le="1"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="2"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="4"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="8"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="16"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="32"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="64"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="128"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="256"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="512"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="1024"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="2048"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="4096"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="8192"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="16384"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="32768"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="65536"} 6 +internal_core_search_latency_bucket{type="scalar_latency",le="+Inf"} 6 +internal_core_search_latency_sum{type="scalar_latency"} 0.761541 +internal_core_search_latency_count{type="scalar_latency"} 6 +internal_core_search_latency_bucket{type="scalar_proportion",le="0"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.05"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.1"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.15"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.2"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.25"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.3"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.35"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.4"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.45"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.5"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.55"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.6"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.65"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.7"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.75"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.8"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.85"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.9"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="0.95"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="1"} 0 +internal_core_search_latency_bucket{type="scalar_proportion",le="+Inf"} 0 +internal_core_search_latency_sum{type="scalar_proportion"} 0 +internal_core_search_latency_count{type="scalar_proportion"} 0 +internal_core_search_latency_bucket{type="vector_latency",le="1"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="2"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="4"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="8"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="16"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="32"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="64"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="128"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="256"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="512"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="1024"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="2048"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="4096"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="8192"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="16384"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="32768"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="65536"} 2 +internal_core_search_latency_bucket{type="vector_latency",le="+Inf"} 2 +internal_core_search_latency_sum{type="vector_latency"} 1.549042 +internal_core_search_latency_count{type="vector_latency"} 2 +# HELP internal_mmap_allocated_space_bytes [cpp]mmap allocated space stats +# TYPE internal_mmap_allocated_space_bytes histogram +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1024"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="8192"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="65536"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="262144"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="524288"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.048576e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="4.194304e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="8.388608e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.6777216e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="6.7108864e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.34217728e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="2.68435456e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="5.36870912e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="1.073741824e+09"} 0 +internal_mmap_allocated_space_bytes_bucket{type="anon",le="+Inf"} 0 +internal_mmap_allocated_space_bytes_sum{type="anon"} 0 +internal_mmap_allocated_space_bytes_count{type="anon"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1024"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="8192"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="65536"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="262144"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="524288"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.048576e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="4.194304e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="8.388608e+06"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.6777216e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="6.7108864e+07"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.34217728e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="2.68435456e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="5.36870912e+08"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="1.073741824e+09"} 0 +internal_mmap_allocated_space_bytes_bucket{type="file",le="+Inf"} 0 +internal_mmap_allocated_space_bytes_sum{type="file"} 0 +internal_mmap_allocated_space_bytes_count{type="file"} 0 +# HELP internal_mmap_in_used_count [cpp]mmap in used count stats +# TYPE internal_mmap_in_used_count gauge +internal_mmap_in_used_count{type="anon"} 0 +internal_mmap_in_used_count{type="file"} 0 +# HELP internal_mmap_in_used_space_bytes [cpp]mmap in used space stats +# TYPE internal_mmap_in_used_space_bytes gauge +internal_mmap_in_used_space_bytes{type="anon"} 0 +internal_mmap_in_used_space_bytes{type="file"} 0 +# HELP internal_storage_kv_size [cpp]kv size stats +# TYPE internal_storage_kv_size histogram +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="1"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="2"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="4"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="8"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="16"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="32"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="64"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="128"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="256"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="512"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="1024"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="2048"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="4096"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="8192"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="16384"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="32768"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="65536"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="get",le="+Inf"} 0 +internal_storage_kv_size_sum{persistent_data_op_type="get"} 0 +internal_storage_kv_size_count{persistent_data_op_type="get"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="1"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="2"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="4"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="8"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="16"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="32"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="64"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="128"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="256"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="512"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="1024"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="2048"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="4096"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="8192"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="16384"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="32768"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="65536"} 0 +internal_storage_kv_size_bucket{persistent_data_op_type="put",le="+Inf"} 0 +internal_storage_kv_size_sum{persistent_data_op_type="put"} 0 +internal_storage_kv_size_count{persistent_data_op_type="put"} 0 +# HELP internal_storage_load_duration [cpp]durations of load segment +# TYPE internal_storage_load_duration histogram +internal_storage_load_duration_bucket{type="deserialize",le="1"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="2"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="4"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="8"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="16"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="32"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="64"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="128"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="256"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="512"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="1024"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="2048"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="4096"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="8192"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="16384"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="32768"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="65536"} 0 +internal_storage_load_duration_bucket{type="deserialize",le="+Inf"} 0 +internal_storage_load_duration_sum{type="deserialize"} 0 +internal_storage_load_duration_count{type="deserialize"} 0 +internal_storage_load_duration_bucket{type="download",le="1"} 0 +internal_storage_load_duration_bucket{type="download",le="2"} 0 +internal_storage_load_duration_bucket{type="download",le="4"} 0 +internal_storage_load_duration_bucket{type="download",le="8"} 0 +internal_storage_load_duration_bucket{type="download",le="16"} 0 +internal_storage_load_duration_bucket{type="download",le="32"} 0 +internal_storage_load_duration_bucket{type="download",le="64"} 0 +internal_storage_load_duration_bucket{type="download",le="128"} 0 +internal_storage_load_duration_bucket{type="download",le="256"} 0 +internal_storage_load_duration_bucket{type="download",le="512"} 0 +internal_storage_load_duration_bucket{type="download",le="1024"} 0 +internal_storage_load_duration_bucket{type="download",le="2048"} 0 +internal_storage_load_duration_bucket{type="download",le="4096"} 0 +internal_storage_load_duration_bucket{type="download",le="8192"} 0 +internal_storage_load_duration_bucket{type="download",le="16384"} 0 +internal_storage_load_duration_bucket{type="download",le="32768"} 0 +internal_storage_load_duration_bucket{type="download",le="65536"} 0 +internal_storage_load_duration_bucket{type="download",le="+Inf"} 0 +internal_storage_load_duration_sum{type="download"} 0 +internal_storage_load_duration_count{type="download"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="1"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="2"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="4"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="8"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="16"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="32"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="64"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="128"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="256"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="512"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="1024"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="2048"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="4096"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="8192"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="16384"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="32768"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="65536"} 0 +internal_storage_load_duration_bucket{type="write_disk",le="+Inf"} 0 +internal_storage_load_duration_sum{type="write_disk"} 0 +internal_storage_load_duration_count{type="write_disk"} 0 +# HELP internal_storage_op_count [cpp]count of persistent data operation +# TYPE internal_storage_op_count counter +internal_storage_op_count{persistent_data_op_type="get"} 0 +internal_storage_op_count{persistent_data_op_type="get",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="list",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="list",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="put",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="put",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="remove",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="remove",status="success"} 0 +internal_storage_op_count{persistent_data_op_type="stat",status="fail"} 0 +internal_storage_op_count{persistent_data_op_type="stat",status="success"} 0 +# HELP internal_storage_request_latency [cpp]request latency(ms) on the client side +# TYPE internal_storage_request_latency histogram +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="get",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="get"} 0 +internal_storage_request_latency_count{persistent_data_op_type="get"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="list",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="list"} 0 +internal_storage_request_latency_count{persistent_data_op_type="list"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="put",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="put"} 0 +internal_storage_request_latency_count{persistent_data_op_type="put"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="remove",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="remove"} 0 +internal_storage_request_latency_count{persistent_data_op_type="remove"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="1"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="2"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="4"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="8"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="16"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="32"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="64"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="128"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="256"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="512"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="1024"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="2048"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="4096"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="8192"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="16384"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="32768"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="65536"} 0 +internal_storage_request_latency_bucket{persistent_data_op_type="stat",le="+Inf"} 0 +internal_storage_request_latency_sum{persistent_data_op_type="stat"} 0 +internal_storage_request_latency_count{persistent_data_op_type="stat"} 0 +# HELP io_cnt io cnt per request +# TYPE io_cnt histogram +io_cnt_bucket{module="cardinal",le="1"} 0 +io_cnt_bucket{module="cardinal",le="2"} 0 +io_cnt_bucket{module="cardinal",le="4"} 0 +io_cnt_bucket{module="cardinal",le="8"} 0 +io_cnt_bucket{module="cardinal",le="16"} 0 +io_cnt_bucket{module="cardinal",le="32"} 0 +io_cnt_bucket{module="cardinal",le="64"} 0 +io_cnt_bucket{module="cardinal",le="128"} 0 +io_cnt_bucket{module="cardinal",le="256"} 0 +io_cnt_bucket{module="cardinal",le="512"} 0 +io_cnt_bucket{module="cardinal",le="1024"} 0 +io_cnt_bucket{module="cardinal",le="2048"} 0 +io_cnt_bucket{module="cardinal",le="4096"} 0 +io_cnt_bucket{module="cardinal",le="8192"} 0 +io_cnt_bucket{module="cardinal",le="16384"} 0 +io_cnt_bucket{module="cardinal",le="32768"} 0 +io_cnt_bucket{module="cardinal",le="65536"} 0 +io_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +io_cnt_bucket{module="cardinal",le="+Inf"} 0 +io_cnt_sum{module="cardinal"} 0 +io_cnt_count{module="cardinal"} 0 +# HELP ivf_search_cnt number of ivf search per request +# TYPE ivf_search_cnt histogram +ivf_search_cnt_bucket{module="cardinal",le="1"} 0 +ivf_search_cnt_bucket{module="cardinal",le="2"} 0 +ivf_search_cnt_bucket{module="cardinal",le="4"} 0 +ivf_search_cnt_bucket{module="cardinal",le="8"} 0 +ivf_search_cnt_bucket{module="cardinal",le="16"} 0 +ivf_search_cnt_bucket{module="cardinal",le="32"} 0 +ivf_search_cnt_bucket{module="cardinal",le="64"} 0 +ivf_search_cnt_bucket{module="cardinal",le="128"} 0 +ivf_search_cnt_bucket{module="cardinal",le="256"} 0 +ivf_search_cnt_bucket{module="cardinal",le="512"} 0 +ivf_search_cnt_bucket{module="cardinal",le="1024"} 0 +ivf_search_cnt_bucket{module="cardinal",le="2048"} 0 +ivf_search_cnt_bucket{module="cardinal",le="4096"} 0 +ivf_search_cnt_bucket{module="cardinal",le="8192"} 0 +ivf_search_cnt_bucket{module="cardinal",le="16384"} 0 +ivf_search_cnt_bucket{module="cardinal",le="32768"} 0 +ivf_search_cnt_bucket{module="cardinal",le="65536"} 0 +ivf_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +ivf_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +ivf_search_cnt_sum{module="cardinal"} 0 +ivf_search_cnt_count{module="cardinal"} 0 +# HELP load_latency index load latency (ms) +# TYPE load_latency histogram +load_latency_bucket{module="cardinal",le="1"} 0 +load_latency_bucket{module="cardinal",le="2"} 0 +load_latency_bucket{module="cardinal",le="4"} 0 +load_latency_bucket{module="cardinal",le="8"} 0 +load_latency_bucket{module="cardinal",le="16"} 0 +load_latency_bucket{module="cardinal",le="32"} 0 +load_latency_bucket{module="cardinal",le="64"} 0 +load_latency_bucket{module="cardinal",le="128"} 0 +load_latency_bucket{module="cardinal",le="256"} 0 +load_latency_bucket{module="cardinal",le="512"} 0 +load_latency_bucket{module="cardinal",le="1024"} 0 +load_latency_bucket{module="cardinal",le="2048"} 0 +load_latency_bucket{module="cardinal",le="4096"} 0 +load_latency_bucket{module="cardinal",le="8192"} 0 +load_latency_bucket{module="cardinal",le="16384"} 0 +load_latency_bucket{module="cardinal",le="32768"} 0 +load_latency_bucket{module="cardinal",le="65536"} 0 +load_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +load_latency_bucket{module="cardinal",le="+Inf"} 0 +load_latency_sum{module="cardinal"} 0 +load_latency_count{module="cardinal"} 0 +load_latency_bucket{module="knowhere",le="1"} 1 +load_latency_bucket{module="knowhere",le="2"} 1 +load_latency_bucket{module="knowhere",le="4"} 1 +load_latency_bucket{module="knowhere",le="8"} 1 +load_latency_bucket{module="knowhere",le="16"} 1 +load_latency_bucket{module="knowhere",le="32"} 1 +load_latency_bucket{module="knowhere",le="64"} 1 +load_latency_bucket{module="knowhere",le="128"} 1 +load_latency_bucket{module="knowhere",le="256"} 1 +load_latency_bucket{module="knowhere",le="512"} 1 +load_latency_bucket{module="knowhere",le="1024"} 1 +load_latency_bucket{module="knowhere",le="2048"} 1 +load_latency_bucket{module="knowhere",le="4096"} 1 +load_latency_bucket{module="knowhere",le="8192"} 1 +load_latency_bucket{module="knowhere",le="16384"} 1 +load_latency_bucket{module="knowhere",le="32768"} 1 +load_latency_bucket{module="knowhere",le="65536"} 1 +load_latency_bucket{module="knowhere",le="1.048576e+06"} 1 +load_latency_bucket{module="knowhere",le="+Inf"} 1 +load_latency_sum{module="knowhere"} 0.1285 +load_latency_count{module="knowhere"} 1 +# HELP quant_compute_cnt quant compute cnt per request +# TYPE quant_compute_cnt histogram +quant_compute_cnt_bucket{module="cardinal",le="1"} 0 +quant_compute_cnt_bucket{module="cardinal",le="2"} 0 +quant_compute_cnt_bucket{module="cardinal",le="4"} 0 +quant_compute_cnt_bucket{module="cardinal",le="8"} 0 +quant_compute_cnt_bucket{module="cardinal",le="16"} 0 +quant_compute_cnt_bucket{module="cardinal",le="32"} 0 +quant_compute_cnt_bucket{module="cardinal",le="64"} 0 +quant_compute_cnt_bucket{module="cardinal",le="128"} 0 +quant_compute_cnt_bucket{module="cardinal",le="256"} 0 +quant_compute_cnt_bucket{module="cardinal",le="512"} 0 +quant_compute_cnt_bucket{module="cardinal",le="1024"} 0 +quant_compute_cnt_bucket{module="cardinal",le="2048"} 0 +quant_compute_cnt_bucket{module="cardinal",le="4096"} 0 +quant_compute_cnt_bucket{module="cardinal",le="8192"} 0 +quant_compute_cnt_bucket{module="cardinal",le="16384"} 0 +quant_compute_cnt_bucket{module="cardinal",le="32768"} 0 +quant_compute_cnt_bucket{module="cardinal",le="65536"} 0 +quant_compute_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +quant_compute_cnt_bucket{module="cardinal",le="+Inf"} 0 +quant_compute_cnt_sum{module="cardinal"} 0 +quant_compute_cnt_count{module="cardinal"} 0 +# HELP queue_latency queue latency per request +# TYPE queue_latency histogram +queue_latency_bucket{module="cardinal",le="1"} 0 +queue_latency_bucket{module="cardinal",le="2"} 0 +queue_latency_bucket{module="cardinal",le="4"} 0 +queue_latency_bucket{module="cardinal",le="8"} 0 +queue_latency_bucket{module="cardinal",le="16"} 0 +queue_latency_bucket{module="cardinal",le="32"} 0 +queue_latency_bucket{module="cardinal",le="64"} 0 +queue_latency_bucket{module="cardinal",le="128"} 0 +queue_latency_bucket{module="cardinal",le="256"} 0 +queue_latency_bucket{module="cardinal",le="512"} 0 +queue_latency_bucket{module="cardinal",le="1024"} 0 +queue_latency_bucket{module="cardinal",le="2048"} 0 +queue_latency_bucket{module="cardinal",le="4096"} 0 +queue_latency_bucket{module="cardinal",le="8192"} 0 +queue_latency_bucket{module="cardinal",le="16384"} 0 +queue_latency_bucket{module="cardinal",le="32768"} 0 +queue_latency_bucket{module="cardinal",le="65536"} 0 +queue_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +queue_latency_bucket{module="cardinal",le="+Inf"} 0 +queue_latency_sum{module="cardinal"} 0 +queue_latency_count{module="cardinal"} 0 +# HELP range_search_latency range search latency (ms) +# TYPE range_search_latency histogram +range_search_latency_bucket{module="cardinal",le="1"} 0 +range_search_latency_bucket{module="cardinal",le="2"} 0 +range_search_latency_bucket{module="cardinal",le="4"} 0 +range_search_latency_bucket{module="cardinal",le="8"} 0 +range_search_latency_bucket{module="cardinal",le="16"} 0 +range_search_latency_bucket{module="cardinal",le="32"} 0 +range_search_latency_bucket{module="cardinal",le="64"} 0 +range_search_latency_bucket{module="cardinal",le="128"} 0 +range_search_latency_bucket{module="cardinal",le="256"} 0 +range_search_latency_bucket{module="cardinal",le="512"} 0 +range_search_latency_bucket{module="cardinal",le="1024"} 0 +range_search_latency_bucket{module="cardinal",le="2048"} 0 +range_search_latency_bucket{module="cardinal",le="4096"} 0 +range_search_latency_bucket{module="cardinal",le="8192"} 0 +range_search_latency_bucket{module="cardinal",le="16384"} 0 +range_search_latency_bucket{module="cardinal",le="32768"} 0 +range_search_latency_bucket{module="cardinal",le="65536"} 0 +range_search_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +range_search_latency_bucket{module="cardinal",le="+Inf"} 0 +range_search_latency_sum{module="cardinal"} 0 +range_search_latency_count{module="cardinal"} 0 +range_search_latency_bucket{module="knowhere",le="1"} 0 +range_search_latency_bucket{module="knowhere",le="2"} 0 +range_search_latency_bucket{module="knowhere",le="4"} 0 +range_search_latency_bucket{module="knowhere",le="8"} 0 +range_search_latency_bucket{module="knowhere",le="16"} 0 +range_search_latency_bucket{module="knowhere",le="32"} 0 +range_search_latency_bucket{module="knowhere",le="64"} 0 +range_search_latency_bucket{module="knowhere",le="128"} 0 +range_search_latency_bucket{module="knowhere",le="256"} 0 +range_search_latency_bucket{module="knowhere",le="512"} 0 +range_search_latency_bucket{module="knowhere",le="1024"} 0 +range_search_latency_bucket{module="knowhere",le="2048"} 0 +range_search_latency_bucket{module="knowhere",le="4096"} 0 +range_search_latency_bucket{module="knowhere",le="8192"} 0 +range_search_latency_bucket{module="knowhere",le="16384"} 0 +range_search_latency_bucket{module="knowhere",le="32768"} 0 +range_search_latency_bucket{module="knowhere",le="65536"} 0 +range_search_latency_bucket{module="knowhere",le="1.048576e+06"} 0 +range_search_latency_bucket{module="knowhere",le="+Inf"} 0 +range_search_latency_sum{module="knowhere"} 0 +range_search_latency_count{module="knowhere"} 0 +# HELP raw_compute_cnt raw compute cnt per request +# TYPE raw_compute_cnt histogram +raw_compute_cnt_bucket{module="cardinal",le="1"} 0 +raw_compute_cnt_bucket{module="cardinal",le="2"} 0 +raw_compute_cnt_bucket{module="cardinal",le="4"} 0 +raw_compute_cnt_bucket{module="cardinal",le="8"} 0 +raw_compute_cnt_bucket{module="cardinal",le="16"} 0 +raw_compute_cnt_bucket{module="cardinal",le="32"} 0 +raw_compute_cnt_bucket{module="cardinal",le="64"} 0 +raw_compute_cnt_bucket{module="cardinal",le="128"} 0 +raw_compute_cnt_bucket{module="cardinal",le="256"} 0 +raw_compute_cnt_bucket{module="cardinal",le="512"} 0 +raw_compute_cnt_bucket{module="cardinal",le="1024"} 0 +raw_compute_cnt_bucket{module="cardinal",le="2048"} 0 +raw_compute_cnt_bucket{module="cardinal",le="4096"} 0 +raw_compute_cnt_bucket{module="cardinal",le="8192"} 0 +raw_compute_cnt_bucket{module="cardinal",le="16384"} 0 +raw_compute_cnt_bucket{module="cardinal",le="32768"} 0 +raw_compute_cnt_bucket{module="cardinal",le="65536"} 0 +raw_compute_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +raw_compute_cnt_bucket{module="cardinal",le="+Inf"} 0 +raw_compute_cnt_sum{module="cardinal"} 0 +raw_compute_cnt_count{module="cardinal"} 0 +# HELP re_search_cnt number of fallback search per request +# TYPE re_search_cnt histogram +re_search_cnt_bucket{module="cardinal",le="1"} 0 +re_search_cnt_bucket{module="cardinal",le="2"} 0 +re_search_cnt_bucket{module="cardinal",le="4"} 0 +re_search_cnt_bucket{module="cardinal",le="8"} 0 +re_search_cnt_bucket{module="cardinal",le="16"} 0 +re_search_cnt_bucket{module="cardinal",le="32"} 0 +re_search_cnt_bucket{module="cardinal",le="64"} 0 +re_search_cnt_bucket{module="cardinal",le="128"} 0 +re_search_cnt_bucket{module="cardinal",le="256"} 0 +re_search_cnt_bucket{module="cardinal",le="512"} 0 +re_search_cnt_bucket{module="cardinal",le="1024"} 0 +re_search_cnt_bucket{module="cardinal",le="2048"} 0 +re_search_cnt_bucket{module="cardinal",le="4096"} 0 +re_search_cnt_bucket{module="cardinal",le="8192"} 0 +re_search_cnt_bucket{module="cardinal",le="16384"} 0 +re_search_cnt_bucket{module="cardinal",le="32768"} 0 +re_search_cnt_bucket{module="cardinal",le="65536"} 0 +re_search_cnt_bucket{module="cardinal",le="1.048576e+06"} 0 +re_search_cnt_bucket{module="cardinal",le="+Inf"} 0 +re_search_cnt_sum{module="cardinal"} 0 +re_search_cnt_count{module="cardinal"} 0 +# HELP search_latency search latency (ms) +# TYPE search_latency histogram +search_latency_bucket{module="cardinal",le="1"} 0 +search_latency_bucket{module="cardinal",le="2"} 0 +search_latency_bucket{module="cardinal",le="4"} 0 +search_latency_bucket{module="cardinal",le="8"} 0 +search_latency_bucket{module="cardinal",le="16"} 0 +search_latency_bucket{module="cardinal",le="32"} 0 +search_latency_bucket{module="cardinal",le="64"} 0 +search_latency_bucket{module="cardinal",le="128"} 0 +search_latency_bucket{module="cardinal",le="256"} 0 +search_latency_bucket{module="cardinal",le="512"} 0 +search_latency_bucket{module="cardinal",le="1024"} 0 +search_latency_bucket{module="cardinal",le="2048"} 0 +search_latency_bucket{module="cardinal",le="4096"} 0 +search_latency_bucket{module="cardinal",le="8192"} 0 +search_latency_bucket{module="cardinal",le="16384"} 0 +search_latency_bucket{module="cardinal",le="32768"} 0 +search_latency_bucket{module="cardinal",le="65536"} 0 +search_latency_bucket{module="cardinal",le="1.048576e+06"} 0 +search_latency_bucket{module="cardinal",le="+Inf"} 0 +search_latency_sum{module="cardinal"} 0 +search_latency_count{module="cardinal"} 0 +search_latency_bucket{module="knowhere",le="1"} 2 +search_latency_bucket{module="knowhere",le="2"} 2 +search_latency_bucket{module="knowhere",le="4"} 2 +search_latency_bucket{module="knowhere",le="8"} 2 +search_latency_bucket{module="knowhere",le="16"} 2 +search_latency_bucket{module="knowhere",le="32"} 2 +search_latency_bucket{module="knowhere",le="64"} 2 +search_latency_bucket{module="knowhere",le="128"} 2 +search_latency_bucket{module="knowhere",le="256"} 2 +search_latency_bucket{module="knowhere",le="512"} 2 +search_latency_bucket{module="knowhere",le="1024"} 2 +search_latency_bucket{module="knowhere",le="2048"} 2 +search_latency_bucket{module="knowhere",le="4096"} 2 +search_latency_bucket{module="knowhere",le="8192"} 2 +search_latency_bucket{module="knowhere",le="16384"} 2 +search_latency_bucket{module="knowhere",le="32768"} 2 +search_latency_bucket{module="knowhere",le="65536"} 2 +search_latency_bucket{module="knowhere",le="1.048576e+06"} 2 +search_latency_bucket{module="knowhere",le="+Inf"} 2 +search_latency_sum{module="knowhere"} 0.8826250000000001 +search_latency_count{module="knowhere"} 2 +# HELP search_topk search topk +# TYPE search_topk histogram +search_topk_bucket{module="cardinal",le="1"} 0 +search_topk_bucket{module="cardinal",le="2"} 0 +search_topk_bucket{module="cardinal",le="4"} 0 +search_topk_bucket{module="cardinal",le="8"} 0 +search_topk_bucket{module="cardinal",le="16"} 0 +search_topk_bucket{module="cardinal",le="32"} 0 +search_topk_bucket{module="cardinal",le="64"} 0 +search_topk_bucket{module="cardinal",le="128"} 0 +search_topk_bucket{module="cardinal",le="256"} 0 +search_topk_bucket{module="cardinal",le="512"} 0 +search_topk_bucket{module="cardinal",le="1024"} 0 +search_topk_bucket{module="cardinal",le="2048"} 0 +search_topk_bucket{module="cardinal",le="4096"} 0 +search_topk_bucket{module="cardinal",le="8192"} 0 +search_topk_bucket{module="cardinal",le="16384"} 0 +search_topk_bucket{module="cardinal",le="32768"} 0 +search_topk_bucket{module="cardinal",le="65536"} 0 +search_topk_bucket{module="cardinal",le="1.048576e+06"} 0 +search_topk_bucket{module="cardinal",le="+Inf"} 0 +search_topk_sum{module="cardinal"} 0 +search_topk_count{module="cardinal"} 0 +search_topk_bucket{module="knowhere",le="1"} 0 +search_topk_bucket{module="knowhere",le="2"} 0 +search_topk_bucket{module="knowhere",le="4"} 2 +search_topk_bucket{module="knowhere",le="8"} 2 +search_topk_bucket{module="knowhere",le="16"} 2 +search_topk_bucket{module="knowhere",le="32"} 2 +search_topk_bucket{module="knowhere",le="64"} 2 +search_topk_bucket{module="knowhere",le="128"} 2 +search_topk_bucket{module="knowhere",le="256"} 2 +search_topk_bucket{module="knowhere",le="512"} 2 +search_topk_bucket{module="knowhere",le="1024"} 2 +search_topk_bucket{module="knowhere",le="2048"} 2 +search_topk_bucket{module="knowhere",le="4096"} 2 +search_topk_bucket{module="knowhere",le="8192"} 2 +search_topk_bucket{module="knowhere",le="16384"} 2 +search_topk_bucket{module="knowhere",le="32768"} 2 +search_topk_bucket{module="knowhere",le="65536"} 2 +search_topk_bucket{module="knowhere",le="1.048576e+06"} 2 +search_topk_bucket{module="knowhere",le="+Inf"} 2 +search_topk_sum{module="knowhere"} 6 +search_topk_count{module="knowhere"} 2 diff --git a/milvus/tests/test_e2e.py b/milvus/tests/test_e2e.py new file mode 100644 index 0000000000000..058f4713b4cdf --- /dev/null +++ b/milvus/tests/test_e2e.py @@ -0,0 +1,23 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.constants import ServiceCheck + +from . import common + + +@pytest.mark.e2e +def test_check_milvus_e2e(dd_agent_check, instance): + aggregator = dd_agent_check(instance, rate=True) + + for metric, _ in common.STANDALONE_TEST_METRICS.items(): + if metric in [ + 'milvus.datacoord.import_tasks', + 'milvus.datacoord.index.task', + ]: # these metrics need a more complex setup to appear + continue + aggregator.assert_metric(name=metric) + + aggregator.assert_service_check('milvus.openmetrics.health', ServiceCheck.OK) diff --git a/milvus/tests/test_unit.py b/milvus/tests/test_unit.py new file mode 100644 index 0000000000000..cbf2e608c731a --- /dev/null +++ b/milvus/tests/test_unit.py @@ -0,0 +1,43 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from datadog_checks.dev.utils import get_metadata_metrics +from datadog_checks.milvus import MilvusCheck + +from .common import STANDALONE_TEST_METRICS, get_fixture_path + + +def test_check(dd_run_check, aggregator, instance, mock_http_response): + mock_http_response(file_path=get_fixture_path('milvus_payload.txt')) + + check = MilvusCheck('milvus', {}, [instance]) + dd_run_check(check) + + for metric, metric_type in STANDALONE_TEST_METRICS.items(): + aggregator.assert_metric(metric, metric_type=aggregator.METRIC_ENUM_MAP[metric_type]) + aggregator.assert_metric_has_tag(metric, 'test:tag') + + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata(get_metadata_metrics()) + + +def test_empty_instance(dd_run_check): + with pytest.raises( + Exception, + match='InstanceConfig`:\nopenmetrics_endpoint\n Field required', + ): + check = MilvusCheck('milvus', {}, [{}]) + dd_run_check(check) + + +def test_custom_validation(dd_run_check): + endpoint = 'milvus:2112/metrics' + with pytest.raises( + Exception, + match='openmetrics_endpoint: {} is incorrectly configured'.format(endpoint), + ): + check = MilvusCheck('milvus', {}, [{'openmetrics_endpoint': endpoint}]) + dd_run_check(check)