diff --git a/examples/metrics/prometheus.py b/examples/metrics/prometheus.py new file mode 100644 index 00000000000..14f612c6a93 --- /dev/null +++ b/examples/metrics/prometheus.py @@ -0,0 +1,55 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module serves as an example for a simple application using metrics +Examples show how to recording affects the collection of metrics to be exported +""" + +from prometheus_client import start_http_server + +from opentelemetry import metrics +from opentelemetry.ext.prometheus import PrometheusMetricsExporter +from opentelemetry.sdk.metrics import Counter, Meter +from opentelemetry.sdk.metrics.export.controller import PushController + +# Start Prometheus client +start_http_server(port=8000, addr="localhost") + +# Meter is responsible for creating and recording metrics +metrics.set_preferred_meter_implementation(lambda _: Meter()) +meter = metrics.meter() +# exporter to export metrics to Prometheus +prefix = "MyAppPrefix" +exporter = PrometheusMetricsExporter(prefix) +# controller collects metrics created from meter and exports it via the +# exporter every interval +controller = PushController(meter, exporter, 5) + +counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), +) + +# Labelsets are used to identify key-values that are associated with a specific +# metric that you want to record. These are useful for pre-aggregation and can +# be used to store custom dimensions pertaining to a metric +label_set = meter.get_label_set({"environment": "staging"}) + +counter.add(25, label_set) +input("Press any key to exit...") diff --git a/ext/opentelemetry-ext-prometheus/CHANGELOG.md b/ext/opentelemetry-ext-prometheus/CHANGELOG.md new file mode 100644 index 00000000000..617d979ab29 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## Unreleased + diff --git a/ext/opentelemetry-ext-prometheus/README.rst b/ext/opentelemetry-ext-prometheus/README.rst new file mode 100644 index 00000000000..2d968d6a7c8 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/README.rst @@ -0,0 +1,72 @@ +OpenTelemetry Prometheus Exporter +============================= + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-ext-prometheus.svg + :target: https://pypi.org/project/opentelemetry-ext-prometheus/ + +This library allows to export metrics data to `Prometheus `_. + +Installation +------------ + +:: + + pip install opentelemetry-ext-prometheus + + +Usage +----- + +The **OpenTelemetry Prometheus Exporter** allows to export `OpenTelemetry`_ metrics to `Prometheus`_. + + +.. _Prometheus: https://prometheus.io/ +.. _OpenTelemetry: https://github.com/open-telemetry/opentelemetry-python/ + +.. code:: python + + from opentelemetry import metrics + from opentelemetry.ext.prometheus import PrometheusMetricsExporter + from opentelemetry.sdk.metrics import Counter, Meter + from opentelemetry.sdk.metrics.export.controller import PushController + from prometheus_client import start_http_server + + # Start Prometheus client + start_http_server(port=8000, addr="localhost") + + # Meter is responsible for creating and recording metrics + metrics.set_preferred_meter_implementation(lambda _: Meter()) + meter = metrics.meter() + # exporter to export metrics to Prometheus + prefix = "MyAppPrefix" + exporter = PrometheusMetricsExporter(prefix) + # controller collects metrics created from meter and exports it via the + # exporter every interval + controller = PushController(meter, exporter, 5) + + counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), + ) + + # Labelsets are used to identify key-values that are associated with a specific + # metric that you want to record. These are useful for pre-aggregation and can + # be used to store custom dimensions pertaining to a metric + label_set = meter.get_label_set({"environment": "staging"}) + + counter.add(25, label_set) + input("Press any key to exit...") + + + +References +---------- + +* `Prometheus `_ +* `OpenTelemetry Project `_ diff --git a/ext/opentelemetry-ext-prometheus/setup.cfg b/ext/opentelemetry-ext-prometheus/setup.cfg new file mode 100644 index 00000000000..f6bfd7c38b2 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/setup.cfg @@ -0,0 +1,47 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-ext-prometheus +description = Prometheus Metric Exporter for OpenTelemetry +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-ext-prometheus +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 3 - Alpha + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + +[options] +python_requires = >=3.4 +package_dir= + =src +packages=find_namespace: +install_requires = + prometheus_client >= 0.5.0, < 1.0.0 + opentelemetry-api + opentelemetry-sdk + +[options.packages.find] +where = src diff --git a/ext/opentelemetry-ext-prometheus/setup.py b/ext/opentelemetry-ext-prometheus/setup.py new file mode 100644 index 00000000000..aa968af60db --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/setup.py @@ -0,0 +1,26 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "ext", "prometheus", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/__init__.py b/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/__init__.py new file mode 100644 index 00000000000..5b4a17a5569 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/__init__.py @@ -0,0 +1,147 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prometheus Metrics Exporter for OpenTelemetry.""" + +import collections +import logging +import re +from typing import Sequence + +from prometheus_client import start_http_server +from prometheus_client.core import ( + REGISTRY, + CollectorRegistry, + CounterMetricFamily, + GaugeMetricFamily, + UnknownMetricFamily, +) + +from opentelemetry.metrics import Counter, Gauge, Measure, Metric +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExporter, + MetricsExportResult, +) + +logger = logging.getLogger(__name__) + + +class PrometheusMetricsExporter(MetricsExporter): + """Prometheus metric exporter for OpenTelemetry. + + Args: + prefix: single-word application prefix relevant to the domain + the metric belongs to. + """ + + def __init__(self, prefix: str = ""): + self._collector = CustomCollector(prefix) + REGISTRY.register(self._collector) + + def export( + self, metric_records: Sequence[MetricRecord] + ) -> MetricsExportResult: + self._collector.add_metrics_data(metric_records) + return MetricsExportResult.SUCCESS + + def shutdown(self) -> None: + REGISTRY.unregister(self._collector) + + +class CustomCollector: + """ CustomCollector represents the Prometheus Collector object + https://github.com/prometheus/client_python#custom-collectors + """ + + def __init__(self, prefix: str = ""): + self._prefix = prefix + self._metrics_to_export = collections.deque() + self._non_letters_nor_digits_re = re.compile( + r"[^\w]", re.UNICODE | re.IGNORECASE + ) + + def add_metrics_data(self, metric_records: Sequence[MetricRecord]): + self._metrics_to_export.append(metric_records) + + def collect(self): + """Collect fetches the metrics from OpenTelemetry + and delivers them as Prometheus Metrics. + Collect is invoked every time a prometheus.Gatherer is run + for example when the HTTP endpoint is invoked by Prometheus. + """ + + while self._metrics_to_export: + for metric_record in self._metrics_to_export.popleft(): + prometheus_metric = self._translate_to_prometheus( + metric_record + ) + if prometheus_metric is not None: + yield prometheus_metric + + def _translate_to_prometheus(self, metric_record: MetricRecord): + prometheus_metric = None + label_values = [] + label_keys = [] + for label_tuple in metric_record.label_set.labels: + label_keys.append(self._sanitize(label_tuple[0])) + label_values.append(label_tuple[1]) + + metric_name = "" + if self._prefix != "": + metric_name = self._prefix + "_" + metric_name += self._sanitize(metric_record.metric.name) + + if isinstance(metric_record.metric, Counter): + prometheus_metric = CounterMetricFamily( + name=metric_name, + documentation=metric_record.metric.description, + labels=label_keys, + ) + prometheus_metric.add_metric( + labels=label_values, value=metric_record.aggregator.checkpoint + ) + + elif isinstance(metric_record.metric, Gauge): + prometheus_metric = GaugeMetricFamily( + name=metric_name, + documentation=metric_record.metric.description, + labels=label_keys, + ) + prometheus_metric.add_metric( + labels=label_values, value=metric_record.aggregator.checkpoint + ) + + # TODO: Add support for histograms when supported in OT + elif isinstance(metric_record.metric, Measure): + prometheus_metric = UnknownMetricFamily( + name=metric_name, + documentation=metric_record.metric.description, + labels=label_keys, + ) + prometheus_metric.add_metric( + labels=label_values, value=metric_record.aggregator.checkpoint + ) + + else: + logger.warning( + "Unsupported metric type. %s", type(metric_record.metric) + ) + return prometheus_metric + + def _sanitize(self, key): + """ sanitize the given metric name or label according to Prometheus rule. + Replace all characters other than [A-Za-z0-9_] with '_'. + """ + return self._non_letters_nor_digits_re.sub("_", key) diff --git a/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/version.py b/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/version.py new file mode 100644 index 00000000000..6b39cd19b59 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/version.py @@ -0,0 +1,15 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.4.dev0" diff --git a/ext/opentelemetry-ext-prometheus/tests/__init__.py b/ext/opentelemetry-ext-prometheus/tests/__init__.py new file mode 100644 index 00000000000..6ab2e961ec4 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ext/opentelemetry-ext-prometheus/tests/test_prometheus_exporter.py b/ext/opentelemetry-ext-prometheus/tests/test_prometheus_exporter.py new file mode 100644 index 00000000000..94fea96c5b5 --- /dev/null +++ b/ext/opentelemetry-ext-prometheus/tests/test_prometheus_exporter.py @@ -0,0 +1,156 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +from prometheus_client.core import CounterMetricFamily + +from opentelemetry.ext.prometheus import ( + CustomCollector, + PrometheusMetricsExporter, +) +from opentelemetry.sdk import metrics +from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult +from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator + + +class TestPrometheusMetricExporter(unittest.TestCase): + def setUp(self): + self._meter = metrics.Meter() + self._test_metric = self._meter.create_metric( + "testname", + "testdesc", + "unit", + int, + metrics.Counter, + ["environment"], + ) + kvp = {"environment": "staging"} + self._test_label_set = self._meter.get_label_set(kvp) + + self._mock_registry_register = mock.Mock() + self._registry_register_patch = mock.patch( + "prometheus_client.core.REGISTRY.register", + side_effect=self._mock_registry_register, + ) + + # pylint: disable=protected-access + def test_constructor(self): + """Test the constructor.""" + with self._registry_register_patch: + exporter = PrometheusMetricsExporter("testprefix") + self.assertEqual(exporter._collector._prefix, "testprefix") + self.assertTrue(self._mock_registry_register.called) + + def test_shutdown(self): + with mock.patch( + "prometheus_client.core.REGISTRY.unregister" + ) as registry_unregister_patch: + exporter = PrometheusMetricsExporter() + exporter.shutdown() + self.assertTrue(registry_unregister_patch.called) + + def test_export(self): + with self._registry_register_patch: + record = MetricRecord( + CounterAggregator(), self._test_label_set, self._test_metric + ) + exporter = PrometheusMetricsExporter() + result = exporter.export([record]) + # pylint: disable=protected-access + self.assertEqual(len(exporter._collector._metrics_to_export), 1) + self.assertIs(result, MetricsExportResult.SUCCESS) + + def test_counter_to_prometheus(self): + meter = metrics.Meter() + metric = meter.create_metric( + "test@name", + "testdesc", + "unit", + int, + metrics.Counter, + ["environment@", "os"], + ) + kvp = {"environment@": "staging", "os": "Windows"} + label_set = meter.get_label_set(kvp) + aggregator = CounterAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, label_set, metric) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + + for prometheus_metric in collector.collect(): + self.assertEqual(type(prometheus_metric), CounterMetricFamily) + self.assertEqual(prometheus_metric.name, "testprefix_test_name") + self.assertEqual(prometheus_metric.documentation, "testdesc") + self.assertTrue(len(prometheus_metric.samples) == 1) + self.assertEqual(prometheus_metric.samples[0].value, 123) + self.assertTrue(len(prometheus_metric.samples[0].labels) == 2) + self.assertEqual( + prometheus_metric.samples[0].labels["environment_"], "staging" + ) + self.assertEqual( + prometheus_metric.samples[0].labels["os"], "Windows" + ) + + # TODO: Add unit test once GaugeAggregator is available + # TODO: Add unit test once Measure Aggregators are available + + def test_invalid_metric(self): + + meter = metrics.Meter() + metric = meter.create_metric( + "tesname", "testdesc", "unit", int, TestMetric + ) + kvp = {"environment": "staging"} + label_set = meter.get_label_set(kvp) + record = MetricRecord(None, label_set, metric) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + collector.collect() + self.assertLogs("opentelemetry.ext.prometheus", level="WARNING") + + def test_sanitize(self): + collector = CustomCollector("testprefix") + self.assertEqual( + collector._sanitize("1!2@3#4$5%6^7&8*9(0)_-"), + "1_2_3_4_5_6_7_8_9_0___", + ) + self.assertEqual(collector._sanitize(",./?;:[]{}"), "__________") + self.assertEqual(collector._sanitize("TestString"), "TestString") + self.assertEqual(collector._sanitize("aAbBcC_12_oi"), "aAbBcC_12_oi") + + +class TestMetric(metrics.Metric): + def __init__( + self, + name: str, + description: str, + unit: str, + value_type, + meter, + label_keys, + enabled: bool = True, + ): + super().__init__( + name, + description, + unit, + value_type, + meter, + label_keys=label_keys, + enabled=enabled, + ) diff --git a/tox.ini b/tox.ini index 8d5fe1d9fea..be7f1db9f73 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,10 @@ envlist = ; opentelemetry-ext-mysql py3{4,5,6,7,8}-test-ext-mysql pypy3-test-ext-mysql + + ; opentelemetry-ext-prometheus + py3{4,5,6,7,8}-test-ext-prometheus + pypy3-test-ext-prometheus ; opentelemetry-ext-psycopg2 py3{4,5,6,7,8}-test-ext-psycopg2 @@ -99,6 +103,7 @@ changedir = test-ext-jaeger: ext/opentelemetry-ext-jaeger/tests test-ext-dbapi: ext/opentelemetry-ext-dbapi/tests test-ext-mysql: ext/opentelemetry-ext-mysql/tests + test-ext-prometheus: ext/opentelemetry-ext-prometheus/tests test-ext-pymongo: ext/opentelemetry-ext-pymongo/tests test-ext-psycopg2: ext/opentelemetry-ext-psycopg2/tests test-ext-wsgi: ext/opentelemetry-ext-wsgi/tests @@ -135,6 +140,8 @@ commands_pre = dbapi: pip install {toxinidir}/ext/opentelemetry-ext-dbapi mysql: pip install {toxinidir}/ext/opentelemetry-ext-dbapi mysql: pip install {toxinidir}/ext/opentelemetry-ext-mysql + prometheus: pip install {toxinidir}/opentelemetry-sdk + prometheus: pip install {toxinidir}/ext/opentelemetry-ext-prometheus pymongo: pip install {toxinidir}/ext/opentelemetry-ext-pymongo psycopg2: pip install {toxinidir}/ext/opentelemetry-ext-dbapi psycopg2: pip install {toxinidir}/ext/opentelemetry-ext-psycopg2