From 925f1439ce28438bad411220dc9c5b8f421fe5c4 Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 31 Mar 2020 19:24:29 -0700 Subject: [PATCH 01/39] release --- azure_monitor/CHANGELOG.md | 13 ++ azure_monitor/README.md | 169 +++++++++++++++++++++ azure_monitor/setup.cfg | 6 +- azure_monitor/src/azure_monitor/version.py | 2 +- 4 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 azure_monitor/CHANGELOG.md create mode 100644 azure_monitor/README.md diff --git a/azure_monitor/CHANGELOG.md b/azure_monitor/CHANGELOG.md new file mode 100644 index 0000000..ca2f99e --- /dev/null +++ b/azure_monitor/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## Unreleased + +## 0.2.0 +Released 2020-03-31 + +- Initial beta release + +## 0.1.0 +Released 2019-11-06 + +- Initial alpha release diff --git a/azure_monitor/README.md b/azure_monitor/README.md new file mode 100644 index 0000000..112d3ff --- /dev/null +++ b/azure_monitor/README.md @@ -0,0 +1,169 @@ +# OpenTelemetry Azure Monitor SDKs and Exporters + +[![Gitter chat](https://img.shields.io/gitter/room/Microsoft/azure-monitor-python)](https://gitter.im/Microsoft/azure-monitor-python) +[![Build status](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python.svg?branch=master)](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python) +[![PyPI version](https://badge.fury.io/py/opentelemetry-azure-monitor.svg)](https://badge.fury.io/py/opentelemetry-azure-monitor) + +## Installation + +```sh +pip install opentelemetry-azure-monitor +``` + +## Documentation + +The online documentation is available at https://opentelemetry-azure-monitor-python.readthedocs.io/. + + +## Usage + +### Trace + +The **Azure Monitor Trace Exporter** allows you to export [OpenTelemetry](https://opentelemetry.io/) traces to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). + +This example shows how to send a span "hello" to Azure Monitor. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_tracer_provider(TracerProvider()) + +# We tell OpenTelemetry who it is that is creating spans. In this case, we have +# no real name (no setup.py), so we make one up. If we had a version, we would +# also specify it here. +tracer = trace.get_tracer(__name__) + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=', +) + +# SpanExporter receives the spans and send them to the target location. +span_processor = BatchExportSpanProcessor(exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +with tracer.start_as_current_span('hello'): + print('Hello World!') +``` + +#### Integrations + +OpenTelemetry also supports several [integrations](https://github.com/open-telemetry/opentelemetry-python/tree/master/ext) which allows to integrate with third party libraries. + +This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/)_ library. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Install the `requests` integration package using ``pip install opentelemetry-ext-http-requests``. +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +import requests + +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_tracer_provider(TracerProvider()) +tracer_provider = trace.get_tracer_provider() + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=', + ) +span_processor = BatchExportSpanProcessor(exporter) +tracer_provider.add_span_processor(span_processor) + +http_requests.enable(tracer_provider) +response = requests.get(url="https://azure.microsoft.com/") +``` + +#### Modifying Traces + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an [envelope](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py#L80) data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py). +* The `AzureMonitorSpanExporter` handles `Data` data types. + +```python +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +# Callback function to add os_type: linux to span properties +def callback_function(envelope): + envelope.data.baseData.properties['os_type'] = 'linux' + return True + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=' +) +exporter.add_telemetry_processor(callback_function) + +trace.set_tracer_provider(TracerProvider()) +tracer = trace.get_tracer(__name__) +span_processor = BatchExportSpanProcessor(exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +with tracer.start_as_current_span('hello'): + print('Hello World!') +``` + +### Metrics + +The **Azure Monitor Metrics Exporter** allows you to export metrics to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). + +This example shows how to track a counter metric and send it as telemetry every export interval. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +import time + +from azure_monitor import AzureMonitorMetricsExporter +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export.controller import PushController + +metrics.set_meter_provider(MeterProvider()) +meter = metrics.get_meter(__name__) +exporter = AzureMonitorMetricsExporter( + connection_string='InstrumentationKey=' +) +controller = PushController(meter, exporter, 5) + +requests_counter = meter.create_metric( + name="requests", + description="number of requests", + unit="1", + value_type=int, + metric_type=Counter, + label_keys=("environment",), +) + +testing_label_set = meter.get_label_set({"environment": "testing"}) + +requests_counter.add(25, testing_label_set) +time.sleep(100) +``` + +# References + +[Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/) + +[OpenTelemetry Project](https://opentelemetry.io/) + +[OpenTelemetry Python Client](https://github.com/open-telemetry/opentelemetry-python) + +[Azure Monitor Python Gitter](https://gitter.im/Microsoft/azure-monitor-python) diff --git a/azure_monitor/setup.cfg b/azure_monitor/setup.cfg index 0157b80..496120e 100644 --- a/azure_monitor/setup.cfg +++ b/azure_monitor/setup.cfg @@ -2,9 +2,9 @@ # Licensed under the MIT License. [metadata] name = opentelemetry-azure-monitor -description = Azure Monitor integration for OpenTelemetry -long_description = file: README.rst -long_description_content_type = text/x-rst +description = Azure Monitor SDKs and Exporters for OpenTelemetry +long_description = file: README.md +long_description_content_type = text/markdown author = Microsoft author_email = appinsightssdk@microsoft.com url = https://github.com/microsoft/opentelemetry-azure-monitor-python diff --git a/azure_monitor/src/azure_monitor/version.py b/azure_monitor/src/azure_monitor/version.py index 3e3cc8e..835d02b 100644 --- a/azure_monitor/src/azure_monitor/version.py +++ b/azure_monitor/src/azure_monitor/version.py @@ -1,3 +1,3 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -__version__ = "0.2b.0" +__version__ = "0.3.dev0" From 7ddf4f589d35904e03f2186d8b8c60bc673b74b4 Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 31 Mar 2020 19:26:12 -0700 Subject: [PATCH 02/39] remove files --- CHANGELOG.md | 13 ---- README.md | 169 --------------------------------------------------- 2 files changed, 182 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 README.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index ca2f99e..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -## Unreleased - -## 0.2.0 -Released 2020-03-31 - -- Initial beta release - -## 0.1.0 -Released 2019-11-06 - -- Initial alpha release diff --git a/README.md b/README.md deleted file mode 100644 index 112d3ff..0000000 --- a/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# OpenTelemetry Azure Monitor SDKs and Exporters - -[![Gitter chat](https://img.shields.io/gitter/room/Microsoft/azure-monitor-python)](https://gitter.im/Microsoft/azure-monitor-python) -[![Build status](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python.svg?branch=master)](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python) -[![PyPI version](https://badge.fury.io/py/opentelemetry-azure-monitor.svg)](https://badge.fury.io/py/opentelemetry-azure-monitor) - -## Installation - -```sh -pip install opentelemetry-azure-monitor -``` - -## Documentation - -The online documentation is available at https://opentelemetry-azure-monitor-python.readthedocs.io/. - - -## Usage - -### Trace - -The **Azure Monitor Trace Exporter** allows you to export [OpenTelemetry](https://opentelemetry.io/) traces to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). - -This example shows how to send a span "hello" to Azure Monitor. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -trace.set_tracer_provider(TracerProvider()) - -# We tell OpenTelemetry who it is that is creating spans. In this case, we have -# no real name (no setup.py), so we make one up. If we had a version, we would -# also specify it here. -tracer = trace.get_tracer(__name__) - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=', -) - -# SpanExporter receives the spans and send them to the target location. -span_processor = BatchExportSpanProcessor(exporter) -trace.get_tracer_provider().add_span_processor(span_processor) - -with tracer.start_as_current_span('hello'): - print('Hello World!') -``` - -#### Integrations - -OpenTelemetry also supports several [integrations](https://github.com/open-telemetry/opentelemetry-python/tree/master/ext) which allows to integrate with third party libraries. - -This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/)_ library. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Install the `requests` integration package using ``pip install opentelemetry-ext-http-requests``. -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -import requests - -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -trace.set_tracer_provider(TracerProvider()) -tracer_provider = trace.get_tracer_provider() - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=', - ) -span_processor = BatchExportSpanProcessor(exporter) -tracer_provider.add_span_processor(span_processor) - -http_requests.enable(tracer_provider) -response = requests.get(url="https://azure.microsoft.com/") -``` - -#### Modifying Traces - -* You can pass a callback function to the exporter to process telemetry before it is exported. -* Your callback function can return `False` if you do not want this envelope exported. -* Your callback function must accept an [envelope](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py#L80) data type as its parameter. -* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py). -* The `AzureMonitorSpanExporter` handles `Data` data types. - -```python -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -# Callback function to add os_type: linux to span properties -def callback_function(envelope): - envelope.data.baseData.properties['os_type'] = 'linux' - return True - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=' -) -exporter.add_telemetry_processor(callback_function) - -trace.set_tracer_provider(TracerProvider()) -tracer = trace.get_tracer(__name__) -span_processor = BatchExportSpanProcessor(exporter) -trace.get_tracer_provider().add_span_processor(span_processor) - -with tracer.start_as_current_span('hello'): - print('Hello World!') -``` - -### Metrics - -The **Azure Monitor Metrics Exporter** allows you to export metrics to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). - -This example shows how to track a counter metric and send it as telemetry every export interval. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -import time - -from azure_monitor import AzureMonitorMetricsExporter -from opentelemetry import metrics -from opentelemetry.sdk.metrics import Counter, MeterProvider -from opentelemetry.sdk.metrics.export.controller import PushController - -metrics.set_meter_provider(MeterProvider()) -meter = metrics.get_meter(__name__) -exporter = AzureMonitorMetricsExporter( - connection_string='InstrumentationKey=' -) -controller = PushController(meter, exporter, 5) - -requests_counter = meter.create_metric( - name="requests", - description="number of requests", - unit="1", - value_type=int, - metric_type=Counter, - label_keys=("environment",), -) - -testing_label_set = meter.get_label_set({"environment": "testing"}) - -requests_counter.add(25, testing_label_set) -time.sleep(100) -``` - -# References - -[Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/) - -[OpenTelemetry Project](https://opentelemetry.io/) - -[OpenTelemetry Python Client](https://github.com/open-telemetry/opentelemetry-python) - -[Azure Monitor Python Gitter](https://gitter.im/Microsoft/azure-monitor-python) From bb3b1931ae88fa4da08413237f205c5261b1019b Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 1 Apr 2020 12:10:50 -0700 Subject: [PATCH 03/39] fix exporters --- azure_monitor/examples/traces/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_monitor/examples/traces/README.md b/azure_monitor/examples/traces/README.md index 383bf92..07def03 100644 --- a/azure_monitor/examples/traces/README.md +++ b/azure_monitor/examples/traces/README.md @@ -2,7 +2,7 @@ ## Installation ```sh -$ pip install opentelemetry-azure-monitor-exporter +$ pip install opentelemetry-azure-monitor ``` ## Run the Applications From a725808a40356f75d5f11a20dfcbf27d49023be8 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 1 Apr 2020 12:14:32 -0700 Subject: [PATCH 04/39] exporers --- docs/conf.py | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c21be60..b503d5f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ author = "Microsoft" # The full version, including alpha/beta/rc tags -release = "0.0.1b" +release = "0.2b.0" # -- General configuration --------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index e360c22..7526b12 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ The package is available on PyPI, and can installed via pip: .. code-block:: sh - pip install opentelemetry-azure-monitor-exporter + pip install opentelemetry-azure-monitor .. toctree:: :maxdepth: 1 From d4cc2c20b88e32d73d18f40396ee07d4f762d56c Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 2 Apr 2020 09:07:00 -0700 Subject: [PATCH 05/39] max storage --- azure_monitor/src/azure_monitor/options.py | 2 +- azure_monitor/src/azure_monitor/storage.py | 31 +++++++++++++++- azure_monitor/tests/test_storage.py | 42 ++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/azure_monitor/src/azure_monitor/options.py b/azure_monitor/src/azure_monitor/options.py index a8e6c91..739ad37 100644 --- a/azure_monitor/src/azure_monitor/options.py +++ b/azure_monitor/src/azure_monitor/options.py @@ -50,7 +50,7 @@ def __init__( connection_string: str = None, instrumentation_key: str = None, storage_maintenance_period: int = 60, - storage_max_size: int = 100 * 1024 * 1024, + storage_max_size: int = 50 * 1024 * 1024, storage_path: str = None, storage_retention_period: int = 7 * 24 * 60 * 60, timeout: int = 10.0, # networking timeout in seconds diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index f8a265a..d01c1d8 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -3,11 +3,14 @@ import datetime import json +import logging import os import random from azure_monitor.utils import PeriodicTask +logger = logging.getLogger(__name__) + def _fmt(timestamp): return timestamp.strftime("%Y-%m-%dT%H%M%S.%f") @@ -81,7 +84,7 @@ class LocalFileStorage: def __init__( self, path, - max_size=100 * 1024 * 1024, # 100MB + max_size=50 * 1024 * 1024, # 50MiB maintenance_period=60, # 1 minute retention_period=7 * 24 * 60 * 60, # 7 days write_timeout=60, # 1 minute @@ -167,6 +170,8 @@ def get(self): return None def put(self, data, lease_period=0, silent=False): + if not self._check_storage_size(): + return None blob = LocalFileBlob( os.path.join( self.path, @@ -179,3 +184,27 @@ def put(self, data, lease_period=0, silent=False): ) ) return blob.put(data, lease_period=lease_period, silent=silent) + + def _check_storage_size(self): + size = 0 + for dirpath, dirnames, filenames in os.walk(self.path): + for f in filenames: + fp = os.path.join(dirpath, f) + # skip if it is symbolic link + if not os.path.islink(fp): + try: + size += os.path.getsize(fp) + except OSError: + logger.error("Path %s does not exist or is " + "inaccessible.", fp) + continue + if size >= self.max_size: + logger.warning( + "Persistent storage max capacity has been " + "reached. Currently at %fKB. Telemetry will be " + "lost. Please consider increasing the value of " + "'storage_max_size' in exporter config.", + format(size/1024) + ) + return False + return True diff --git a/azure_monitor/tests/test_storage.py b/azure_monitor/tests/test_storage.py index a24b49a..c4258b0 100644 --- a/azure_monitor/tests/test_storage.py +++ b/azure_monitor/tests/test_storage.py @@ -108,6 +108,48 @@ def test_put(self): self.assertIsNone(stor.put(test_input, silent=True)) self.assertRaises(Exception, lambda: stor.put(test_input)) + def test_put_max_size(self): + test_input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd')) as stor: + size_mock = mock.Mock() + size_mock.return_value = False + stor._check_storage_size = size_mock + stor.put(test_input) + self.assertEqual(stor.get(), None) + + def test_check_storage_size_full(self): + test_input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd2'), 1) as stor: + stor.put(test_input) + self.assertFalse(stor._check_storage_size()) + + def test_check_storage_size_not_full(self): + test_input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + stor.put(test_input) + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_no_files(self): + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_links(self): + test_input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd4'), 1000) as stor: + stor.put(test_input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_error(self): + test_input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor: + with mock.patch('os.path.getsize', side_effect=throw(OSError)): + stor.put(test_input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + def test_maintanence_routine(self): with mock.patch("os.makedirs") as m: m.return_value = None From 0f337c30d3a482237292d1b9e5091582998e477a Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 2 Apr 2020 18:40:52 -0700 Subject: [PATCH 06/39] remove label sets --- .../examples/metrics/auto_collector.py | 4 +- azure_monitor/examples/metrics/client.py | 26 ---------- azure_monitor/examples/metrics/observer.py | 47 +++++++++++++++++++ azure_monitor/examples/metrics/simple.py | 4 +- .../azure_monitor/export/metrics/__init__.py | 8 ++-- .../sdk/auto_collection/__init__.py | 13 ++--- .../sdk/auto_collection/dependency_metrics.py | 12 ++--- .../auto_collection/performance_metrics.py | 16 +++---- .../sdk/auto_collection/request_metrics.py | 17 +++---- .../auto_collection/test_auto_collection.py | 11 ++--- .../test_dependency_metrics.py | 19 ++++---- .../test_performance_metrics.py | 27 +++++------ .../auto_collection/test_request_metrics.py | 27 +++++------ azure_monitor/tests/metrics/test_metrics.py | 23 ++++----- 14 files changed, 137 insertions(+), 117 deletions(-) delete mode 100644 azure_monitor/examples/metrics/client.py create mode 100644 azure_monitor/examples/metrics/observer.py diff --git a/azure_monitor/examples/metrics/auto_collector.py b/azure_monitor/examples/metrics/auto_collector.py index cd899d7..75b1230 100644 --- a/azure_monitor/examples/metrics/auto_collector.py +++ b/azure_monitor/examples/metrics/auto_collector.py @@ -14,10 +14,10 @@ ) controller = PushController(meter, exporter, 5) -testing_label_set = meter.get_label_set({"environment": "testing"}) +testing_label_set = {"environment": "testing"} # Automatically collect standard metrics -auto_collection = AutoCollection(meter=meter, label_set=testing_label_set) +auto_collection = AutoCollection(meter=meter, labels=testing_label_set) # To configure a separate export interval specific for standard metrics # meter_standard = metrics.get_meter(__name__ + "_standard") diff --git a/azure_monitor/examples/metrics/client.py b/azure_monitor/examples/metrics/client.py deleted file mode 100644 index 57bec03..0000000 --- a/azure_monitor/examples/metrics/client.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# pylint: disable=import-error -# pylint: disable=no-member -# pylint: disable=no-name-in-module -import requests -from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -from azure_monitor import AzureMonitorSpanExporter - -trace.set_tracer_provider(TracerProvider()) -tracer = trace.get_tracer(__name__) -http_requests.enable(trace.get_tracer_provider()) -span_processor = BatchExportSpanProcessor( - AzureMonitorSpanExporter( - connection_string="InstrumentationKey=" - ) -) -trace.get_tracer_provider().add_span_processor(span_processor) - -response = requests.get(url="http://google.com") - -input("Press any key to exit...") diff --git a/azure_monitor/examples/metrics/observer.py b/azure_monitor/examples/metrics/observer.py new file mode 100644 index 0000000..4f9344f --- /dev/null +++ b/azure_monitor/examples/metrics/observer.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import psutil + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export.controller import PushController + +from azure_monitor import AzureMonitorMetricsExporter + +metrics.set_meter_provider(MeterProvider()) +meter = metrics.get_meter(__name__) +exporter = AzureMonitorMetricsExporter( + connection_string="InstrumentationKey=" +) +controller = PushController(meter=meter, exporter=exporter, interval=2) + +# Callback to gather cpu usage +def get_cpu_usage_callback(observer): + for (number, percent) in enumerate(psutil.cpu_percent(percpu=True)): + labels = {"cpu_number": str(number)} + observer.observe(percent, labels) + +meter.register_observer( + callback=get_cpu_usage_callback, + name="cpu_percent", + description="per-cpu usage", + unit="1", + value_type=float, + label_keys=("cpu_number",), +) + +# Callback to gather RAM memory usage +def get_ram_usage_callback(observer): + ram_percent = psutil.virtual_memory().percent + observer.observe(ram_percent, {}) + +meter.register_observer( + callback=get_ram_usage_callback, + name="ram_percent", + description="RAM memory usage", + unit="1", + value_type=float, + label_keys=(), +) + +input("Metrics will be printed soon. Press a key to finish...\n") diff --git a/azure_monitor/examples/metrics/simple.py b/azure_monitor/examples/metrics/simple.py index 3547764..5c11f79 100644 --- a/azure_monitor/examples/metrics/simple.py +++ b/azure_monitor/examples/metrics/simple.py @@ -22,8 +22,8 @@ label_keys=("environment",), ) -testing_label_set = meter.get_label_set({"environment": "testing"}) +testing_labels = {"environment": "testing"} -requests_counter.add(25, testing_label_set) +requests_counter.add(25, testing_labels) input("Press any key to exit...") diff --git a/azure_monitor/src/azure_monitor/export/metrics/__init__.py b/azure_monitor/src/azure_monitor/export/metrics/__init__.py index 98d7369..d7b987b 100644 --- a/azure_monitor/src/azure_monitor/export/metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/export/metrics/__init__.py @@ -60,12 +60,12 @@ def _metric_to_envelope( if not metric_record: return None - # TODO: Opentelemetry does not have last updated timestamp for observer - # type metrics yet. + # TODO: Opentelemetry has timestamp for Observer, awaiting release + # TODO: Timestamp info is also moved into aggregators _time = time_ns() if isinstance(metric_record.metric, Metric): _time = metric_record.metric.bind( - metric_record.label_set + dict(metric_record.labels) ).last_update_timestamp envelope = protocol.Envelope( ikey=self.options.instrumentation_key, @@ -93,7 +93,7 @@ def _metric_to_envelope( ) properties = {} - for label_tuple in metric_record.label_set.labels: + for label_tuple in metric_record.labels: properties[label_tuple[0]] = label_tuple[1] data = protocol.MetricData(metrics=[data_point], properties=properties) envelope.data = protocol.Data(base_data=data, base_type="MetricData") diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py index b7ebe41..aade45d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py @@ -1,7 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -from opentelemetry.metrics import LabelSet, Meter +from typing import Dict +from opentelemetry.metrics import Meter from azure_monitor.sdk.auto_collection.dependency_metrics import ( DependencyMetrics, @@ -25,10 +26,10 @@ class AutoCollection: Args: meter: OpenTelemetry Meter - label_set: OpenTelemetry label set + labels: Dictionary of labels """ - def __init__(self, meter: Meter, label_set: LabelSet): - self._performance_metrics = PerformanceMetrics(meter, label_set) - self._dependency_metrics = DependencyMetrics(meter, label_set) - self._request_metrics = RequestMetrics(meter, label_set) + def __init__(self, meter: Meter, labels: Dict[str, str]): + self._performance_metrics = PerformanceMetrics(meter, labels) + self._dependency_metrics = DependencyMetrics(meter, labels) + self._request_metrics = RequestMetrics(meter, labels) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py index 4b86a13..476a70e 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py @@ -4,9 +4,9 @@ import time import requests +from typing import Dict from opentelemetry import context from opentelemetry.metrics import Meter -from opentelemetry.sdk.metrics import LabelSet dependency_map = dict() _dependency_lock = threading.Lock() @@ -30,12 +30,12 @@ class DependencyMetrics: Args: meter: OpenTelemetry Meter - label_set: OpenTelemetry label set + labels: Dictionary of labels """ - def __init__(self, meter: Meter, label_set: LabelSet): + def __init__(self, meter: Meter, labels: Dict[str, str]): self._meter = meter - self._label_set = label_set + self._labels = labels # Patch requests requests.Session.request = dependency_patch meter.register_observer( @@ -70,8 +70,8 @@ def _track_dependency_rate(self, observer) -> None: dependency_map["last_time"] = current_time dependency_map["last_count"] = current_count dependency_map["last_result"] = result - observer.observe(int(result), self._label_set) + observer.observe(int(result), self._labels) except ZeroDivisionError: # If elapsed_seconds is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe(int(last_result), self._label_set) + observer.observe(int(last_result), self._labels) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py index 0a66fce..2394727 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py @@ -3,8 +3,8 @@ import logging import psutil +from typing import Dict from opentelemetry.metrics import Meter -from opentelemetry.sdk.metrics import LabelSet logger = logging.getLogger(__name__) PROCESS = psutil.Process() @@ -18,12 +18,12 @@ class PerformanceMetrics: Args: meter: OpenTelemetry Meter - label_set: OpenTelemetry label set + labels: Dictionary of labels """ - def __init__(self, meter: Meter, label_set: LabelSet): + def __init__(self, meter: Meter, labels: Dict[str, str]): self._meter = meter - self._label_set = label_set + self._labels = labels # Create performance metrics meter.register_observer( callback=self._track_cpu, @@ -63,7 +63,7 @@ def _track_cpu(self, observer) -> None: from 0.0 to 100.0 inclusive. """ cpu_times_percent = psutil.cpu_times_percent() - observer.observe(100.0 - cpu_times_percent.idle, self._label_set) + observer.observe(100.0 - cpu_times_percent.idle, self._labels) def _track_memory(self, observer) -> None: """ Track Memory @@ -71,7 +71,7 @@ def _track_memory(self, observer) -> None: Available memory is defined as memory that can be given instantly to processes without the system going into swap. """ - observer.observe(psutil.virtual_memory().available, self._label_set) + observer.observe(psutil.virtual_memory().available, self._labels) def _track_process_cpu(self, observer) -> None: """ Track Process CPU time @@ -85,7 +85,7 @@ def _track_process_cpu(self, observer) -> None: # normalize the cpu process using the number of logical CPUs cpu_count = psutil.cpu_count(logical=True) observer.observe( - PROCESS.cpu_percent() / cpu_count, self._label_set + PROCESS.cpu_percent() / cpu_count, self._labels ) except Exception: # pylint: disable=broad-except logger.exception("Error handling get process cpu usage.") @@ -97,6 +97,6 @@ def _track_process_memory(self, observer) -> None: processes without the system going into swap. """ try: - observer.observe(PROCESS.memory_info().rss, self._label_set) + observer.observe(PROCESS.memory_info().rss, self._labels) except Exception: # pylint: disable=broad-except logger.exception("Error handling get process private bytes.") diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py index 2505cbe..4c31cac 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py @@ -4,8 +4,9 @@ import threading import time from http.server import HTTPServer +from typing import Dict -from opentelemetry.metrics import LabelSet, Meter +from opentelemetry.metrics import Meter logger = logging.getLogger(__name__) @@ -59,12 +60,12 @@ class RequestMetrics: Args: meter: OpenTelemetry Meter - label_set: OpenTelemetry label set + labels: Dictionary of labels """ - def __init__(self, meter: Meter, label_set: LabelSet): + def __init__(self, meter: Meter, labels: Dict[str, str]): self._meter = meter - self._label_set = label_set + self._labels = labels # Patch the HTTPServer handler to track request information HTTPServer.__init__ = server_patch @@ -102,12 +103,12 @@ def _track_request_duration(self, observer) -> None: requests_map["last_average_duration"] = result requests_map["last_duration"] = requests_map.get("duration", 0) # Convert to milliseconds - observer.observe(int(result * 1000.0), self._label_set) + observer.observe(int(result * 1000.0), self._labels) except ZeroDivisionError: # If interval_count is 0, exporter call made too close to previous # Return the previous result if this is the case observer.observe( - int(last_average_duration * 1000.0), self._label_set + int(last_average_duration * 1000.0), self._labels ) def _track_request_rate(self, observer) -> None: @@ -134,8 +135,8 @@ def _track_request_rate(self, observer) -> None: requests_map["last_time"] = current_time requests_map["last_count"] = requests_map.get("count", 0) requests_map["last_rate"] = result - observer.observe(int(result), self._label_set) + observer.observe(int(result), self._labels) except ZeroDivisionError: # If elapsed_seconds is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe(int(last_rate), self._label_set) + observer.observe(int(last_rate), self._labels) diff --git a/azure_monitor/tests/auto_collection/test_auto_collection.py b/azure_monitor/tests/auto_collection/test_auto_collection.py index 301d249..884b00a 100644 --- a/azure_monitor/tests/auto_collection/test_auto_collection.py +++ b/azure_monitor/tests/auto_collection/test_auto_collection.py @@ -16,8 +16,7 @@ class TestAutoCollection(unittest.TestCase): def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) - kvp = {"environment": "staging"} - cls._test_label_set = cls._meter.get_label_set(kvp) + cls._test_labels = tuple({"environment": "staging"}.items()) @classmethod def tearDownClass(cls): @@ -36,17 +35,17 @@ def test_constructor( self, mock_performance, mock_dependencies, mock_requests ): """Test the constructor.""" - AutoCollection(meter=self._meter, label_set=self._test_label_set) + AutoCollection(meter=self._meter, labels=self._test_labels) self.assertEqual(mock_performance.called, True) self.assertEqual(mock_dependencies.called, True) self.assertEqual(mock_requests.called, True) self.assertEqual(mock_performance.call_args[0][0], self._meter) self.assertEqual( - mock_performance.call_args[0][1], self._test_label_set + mock_performance.call_args[0][1], self._test_labels ) self.assertEqual(mock_dependencies.call_args[0][0], self._meter) self.assertEqual( - mock_dependencies.call_args[0][1], self._test_label_set + mock_dependencies.call_args[0][1], self._test_labels ) self.assertEqual(mock_requests.call_args[0][0], self._meter) - self.assertEqual(mock_requests.call_args[0][1], self._test_label_set) + self.assertEqual(mock_requests.call_args[0][1], self._test_labels) diff --git a/azure_monitor/tests/auto_collection/test_dependency_metrics.py b/azure_monitor/tests/auto_collection/test_dependency_metrics.py index fe273df..8514528 100644 --- a/azure_monitor/tests/auto_collection/test_dependency_metrics.py +++ b/azure_monitor/tests/auto_collection/test_dependency_metrics.py @@ -21,8 +21,7 @@ class TestDependencyMetrics(unittest.TestCase): def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) - kvp = {"environment": "staging"} - cls._test_label_set = cls._meter.get_label_set(kvp) + cls._test_labels = {"environment": "staging"} @classmethod def tearDown(cls): @@ -38,10 +37,10 @@ def setUp(self): def test_constructor(self): mock_meter = mock.Mock() metrics_collector = dependency_metrics.DependencyMetrics( - meter=mock_meter, label_set=self._test_label_set + meter=mock_meter, labels=self._test_labels ) self.assertEqual(metrics_collector._meter, mock_meter) - self.assertEqual(metrics_collector._label_set, self._test_label_set) + self.assertEqual(metrics_collector._labels, self._test_labels) self.assertEqual(mock_meter.register_observer.call_count, 1) mock_meter.register_observer.assert_called_with( callback=metrics_collector._track_dependency_rate, @@ -55,7 +54,7 @@ def test_constructor(self): def test_track_dependency_rate(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) obs = Observer( callback=metrics_collector._track_dependency_rate, @@ -68,13 +67,13 @@ def test_track_dependency_rate(self, time_mock): dependency_metrics.dependency_map["last_time"] = 98 dependency_metrics.dependency_map["count"] = 4 metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 2) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 2) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate_time_none(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) dependency_metrics.dependency_map["last_time"] = None obs = Observer( @@ -86,13 +85,13 @@ def test_track_dependency_rate_time_none(self, time_mock): meter=self._meter, ) metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 0) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate_error(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) dependency_metrics.dependency_map["last_time"] = 100 dependency_metrics.dependency_map["last_result"] = 5 @@ -105,7 +104,7 @@ def test_track_dependency_rate_error(self, time_mock): meter=self._meter, ) metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 5) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 5) @mock.patch( "azure_monitor.sdk.auto_collection.dependency_metrics.ORIGINAL_REQUEST" diff --git a/azure_monitor/tests/auto_collection/test_performance_metrics.py b/azure_monitor/tests/auto_collection/test_performance_metrics.py index 6637913..a0df358 100644 --- a/azure_monitor/tests/auto_collection/test_performance_metrics.py +++ b/azure_monitor/tests/auto_collection/test_performance_metrics.py @@ -24,8 +24,7 @@ class TestPerformanceMetrics(unittest.TestCase): def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) - kvp = {"environment": "staging"} - cls._test_label_set = cls._meter.get_label_set(kvp) + cls._test_labels = {"environment": "staging"} @classmethod def tearDownClass(cls): @@ -34,11 +33,11 @@ def tearDownClass(cls): def test_constructor(self): mock_meter = mock.Mock() performance_metrics_collector = PerformanceMetrics( - meter=mock_meter, label_set=self._test_label_set + meter=mock_meter, labels=self._test_labels ) self.assertEqual(performance_metrics_collector._meter, mock_meter) self.assertEqual( - performance_metrics_collector._label_set, self._test_label_set + performance_metrics_collector._labels, self._test_labels ) self.assertEqual(mock_meter.register_observer.call_count, 4) reg_obs_calls = mock_meter.register_observer.call_args_list @@ -73,7 +72,7 @@ def test_constructor(self): def test_track_cpu(self): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) with mock.patch("psutil.cpu_times_percent") as processor_mock: cpu = collections.namedtuple("cpu", "idle") @@ -89,13 +88,13 @@ def test_track_cpu(self): ) performance_metrics_collector._track_cpu(obs) self.assertEqual( - obs.aggregators[self._test_label_set].current, 5.5 + obs.aggregators[tuple(self._test_labels.items())].current, 5.5 ) @mock.patch("psutil.virtual_memory") def test_track_memory(self, psutil_mock): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) memory = collections.namedtuple("memory", "available") vmem = memory(available=100) @@ -109,7 +108,7 @@ def test_track_memory(self, psutil_mock): meter=self._meter, ) performance_metrics_collector._track_memory(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 100) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 100) @mock.patch("azure_monitor.sdk.auto_collection.performance_metrics.psutil") def test_track_process_cpu(self, psutil_mock): @@ -117,7 +116,7 @@ def test_track_process_cpu(self, psutil_mock): "azure_monitor.sdk.auto_collection.performance_metrics.PROCESS" ) as process_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) process_mock.cpu_percent.return_value = 44.4 psutil_mock.cpu_count.return_value = 2 @@ -131,7 +130,7 @@ def test_track_process_cpu(self, psutil_mock): ) performance_metrics_collector._track_process_cpu(obs) self.assertEqual( - obs.aggregators[self._test_label_set].current, 22.2 + obs.aggregators[tuple(self._test_labels.items())].current, 22.2 ) @mock.patch("azure_monitor.sdk.auto_collection.performance_metrics.logger") @@ -140,7 +139,7 @@ def test_track_process_cpu_exception(self, logger_mock): "azure_monitor.sdk.auto_collection.performance_metrics.psutil" ) as psutil_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) psutil_mock.cpu_count.return_value = None obs = Observer( @@ -159,7 +158,7 @@ def test_track_process_memory(self): "azure_monitor.sdk.auto_collection.performance_metrics.PROCESS" ) as process_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) memory = collections.namedtuple("memory", "rss") pmem = memory(rss=100) @@ -174,7 +173,7 @@ def test_track_process_memory(self): ) performance_metrics_collector._track_process_memory(obs) self.assertEqual( - obs.aggregators[self._test_label_set].current, 100 + obs.aggregators[tuple(self._test_labels.items())].current, 100 ) @mock.patch("azure_monitor.sdk.auto_collection.performance_metrics.logger") @@ -184,7 +183,7 @@ def test_track_process_memory_exception(self, logger_mock): throw(Exception), ): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) obs = Observer( callback=performance_metrics_collector._track_process_memory, diff --git a/azure_monitor/tests/auto_collection/test_request_metrics.py b/azure_monitor/tests/auto_collection/test_request_metrics.py index 63030d1..903e150 100644 --- a/azure_monitor/tests/auto_collection/test_request_metrics.py +++ b/azure_monitor/tests/auto_collection/test_request_metrics.py @@ -21,8 +21,7 @@ class TestRequestMetrics(unittest.TestCase): def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) - kvp = {"environment": "staging"} - cls._test_label_set = cls._meter.get_label_set(kvp) + cls._test_labels = {"environment": "staging"} @classmethod def tearDown(cls): @@ -36,11 +35,11 @@ def setUp(self): def test_constructor(self): mock_meter = mock.Mock() request_metrics_collector = request_metrics.RequestMetrics( - meter=mock_meter, label_set=self._test_label_set + meter=mock_meter, labels=self._test_labels ) self.assertEqual(request_metrics_collector._meter, mock_meter) self.assertEqual( - request_metrics_collector._label_set, self._test_label_set + request_metrics_collector._labels, self._test_labels ) self.assertEqual(mock_meter.register_observer.call_count, 2) @@ -65,7 +64,7 @@ def test_constructor(self): def test_track_request_duration(self): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) request_metrics.requests_map["duration"] = 0.1 request_metrics.requests_map["count"] = 10 @@ -79,11 +78,11 @@ def test_track_request_duration(self): meter=self._meter, ) request_metrics_collector._track_request_duration(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 20) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 20) def test_track_request_duration_error(self): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) request_metrics.requests_map["duration"] = 0.1 request_metrics.requests_map["count"] = 10 @@ -97,12 +96,12 @@ def test_track_request_duration_error(self): meter=self._meter, ) request_metrics_collector._track_request_duration(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 0) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate(self, time_mock): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) time_mock.time.return_value = 100 request_metrics.requests_map["last_time"] = 98 @@ -116,13 +115,13 @@ def test_track_request_rate(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 2) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 2) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_time_none(self, time_mock): time_mock.time.return_value = 100 request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) request_metrics.requests_map["last_time"] = None obs = Observer( @@ -134,12 +133,12 @@ def test_track_request_rate_time_none(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 0) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_error(self, time_mock): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, label_set=self._test_label_set + meter=self._meter, labels=self._test_labels ) time_mock.time.return_value = 100 request_metrics.requests_map["last_rate"] = 5 @@ -153,7 +152,7 @@ def test_track_request_rate_error(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[self._test_label_set].current, 5) + self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 5) def test_request_patch(self): map = request_metrics.requests_map # pylint: disable=redefined-builtin diff --git a/azure_monitor/tests/metrics/test_metrics.py b/azure_monitor/tests/metrics/test_metrics.py index 1984c3b..e2a7baa 100644 --- a/azure_monitor/tests/metrics/test_metrics.py +++ b/azure_monitor/tests/metrics/test_metrics.py @@ -67,8 +67,7 @@ def setUpClass(cls): Counter, ["environment"], ) - kvp = {"environment": "staging"} - cls._test_label_set = cls._meter.get_label_set(kvp) + cls._test_labels = tuple({"environment": "staging"}.items()) def setUp(self): for filename in os.listdir(STORAGE_PATH): @@ -99,7 +98,7 @@ def test_constructor(self): def test_export(self,): record = MetricRecord( - CounterAggregator(), self._test_label_set, self._test_metric + CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter with mock.patch( @@ -111,7 +110,7 @@ def test_export(self,): def test_export_failed_retryable(self): record = MetricRecord( - CounterAggregator(), self._test_label_set, self._test_metric + CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter with mock.patch( @@ -127,7 +126,7 @@ def test_export_failed_retryable(self): @mock.patch("azure_monitor.export.metrics.logger") def test_export_exception(self, logger_mock): record = MetricRecord( - CounterAggregator(), self._test_label_set, self._test_metric + CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter with mock.patch( @@ -147,7 +146,7 @@ def test_metric_to_envelope(self): aggregator.update(123) aggregator.take_checkpoint() record = MetricRecord( - aggregator, self._test_label_set, self._test_metric + aggregator, self._test_labels, self._test_metric ) exporter = self._exporter envelope = exporter._metric_to_envelope(record) @@ -157,7 +156,7 @@ def test_metric_to_envelope(self): self.assertEqual( envelope.time, ns_to_iso_str( - record.metric.bind(record.label_set).last_update_timestamp + record.metric.bind(dict(record.labels)).last_update_timestamp ), ) self.assertEqual(envelope.sample_rate, None) @@ -188,7 +187,8 @@ def test_observer_to_envelope(self): aggregator = ObserverAggregator() aggregator.update(123) aggregator.take_checkpoint() - record = MetricRecord(aggregator, self._test_label_set, self._test_obs) + record = MetricRecord(aggregator, self._test_labels, self._test_obs) + print(record.labels) exporter = self._exporter envelope = exporter._metric_to_envelope(record) self.assertIsInstance(envelope, Envelope) @@ -231,7 +231,7 @@ def test_observer_to_envelope_value_none(self): aggregator = ObserverAggregator() aggregator.update(None) aggregator.take_checkpoint() - record = MetricRecord(aggregator, self._test_label_set, self._test_obs) + record = MetricRecord(aggregator, self._test_labels, self._test_obs) exporter = self._exporter envelope = exporter._metric_to_envelope(record) self.assertIsInstance(envelope, Envelope) @@ -259,6 +259,7 @@ def test_observer_to_envelope_value_none(self): self.assertEqual(envelope.data.base_data.metrics[0].ns, "testdesc") self.assertEqual(envelope.data.base_data.metrics[0].name, "testname") self.assertEqual(envelope.data.base_data.metrics[0].value, 0) + print(envelope.data.base_data.properties) self.assertEqual( envelope.data.base_data.properties["environment"], "staging" ) @@ -276,7 +277,7 @@ def test_measure_to_envelope(self, logger_mock): aggregator.update(123) aggregator.take_checkpoint() record = MetricRecord( - aggregator, self._test_label_set, self._test_measure + aggregator, self._test_labels, self._test_measure ) exporter = self._exporter envelope = exporter._metric_to_envelope(record) @@ -286,7 +287,7 @@ def test_measure_to_envelope(self, logger_mock): self.assertEqual( envelope.time, ns_to_iso_str( - record.metric.bind(record.label_set).last_update_timestamp + record.metric.bind(dict(record.labels)).last_update_timestamp ), ) self.assertEqual(envelope.sample_rate, None) From d218c197e977b012764758464ebb7fa206a4fae8 Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 3 Apr 2020 09:37:42 -0700 Subject: [PATCH 07/39] is remote --- azure_monitor/README.md | 4 ++-- azure_monitor/tests/trace/test_trace.py | 26 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/azure_monitor/README.md b/azure_monitor/README.md index 112d3ff..e31d400 100644 --- a/azure_monitor/README.md +++ b/azure_monitor/README.md @@ -152,9 +152,9 @@ requests_counter = meter.create_metric( label_keys=("environment",), ) -testing_label_set = meter.get_label_set({"environment": "testing"}) +testing_labels = {"environment": "testing"} -requests_counter.add(25, testing_label_set) +requests_counter.add(25, testing_labels) time.sleep(100) ``` diff --git a/azure_monitor/tests/trace/test_trace.py b/azure_monitor/tests/trace/test_trace.py index 3e3f161..44d8853 100644 --- a/azure_monitor/tests/trace/test_trace.py +++ b/azure_monitor/tests/trace/test_trace.py @@ -92,6 +92,7 @@ def test_export_failure(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557338, + is_remote=False, ), ) test_span.start() @@ -108,6 +109,7 @@ def test_export_success(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557338, + is_remote=False, ), ) test_span.start() @@ -129,6 +131,7 @@ def test_export_exception(self, logger_mock): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557338, + is_remote=False, ), ) test_span.start() @@ -149,6 +152,7 @@ def test_export_not_retryable(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557338, + is_remote=False, ), ) test_span.start() @@ -176,6 +180,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557338, + is_remote=False, ), ) @@ -188,6 +193,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -237,6 +243,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -275,6 +282,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -323,6 +331,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -376,6 +385,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -429,6 +439,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -472,6 +483,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=None, sampler=None, @@ -510,6 +522,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -543,6 +556,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195432, span_id=12030755672171557338, + is_remote=False, ) ) ) @@ -551,6 +565,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -582,6 +597,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -609,6 +625,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -636,6 +653,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -662,6 +680,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -688,6 +707,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -714,6 +734,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -741,6 +762,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -777,6 +799,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -804,6 +827,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -840,6 +864,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, @@ -873,6 +898,7 @@ def test_span_to_envelope(self): context=SpanContext( trace_id=36873507687745823477771305566750195431, span_id=12030755672171557337, + is_remote=False, ), parent=parent_span, sampler=None, From 5251aa4ab9392861323632c48f3f202a2c8525a8 Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 3 Apr 2020 09:43:34 -0700 Subject: [PATCH 08/39] lint --- azure_monitor/examples/metrics/observer.py | 3 ++- .../sdk/auto_collection/__init__.py | 1 + .../sdk/auto_collection/dependency_metrics.py | 2 +- .../auto_collection/performance_metrics.py | 6 ++--- .../sdk/auto_collection/request_metrics.py | 4 +--- azure_monitor/src/azure_monitor/storage.py | 7 +++--- .../auto_collection/test_auto_collection.py | 8 ++----- .../test_dependency_metrics.py | 12 +++++++--- .../test_performance_metrics.py | 4 +++- .../auto_collection/test_request_metrics.py | 24 ++++++++++++------- azure_monitor/tests/metrics/test_metrics.py | 4 +--- azure_monitor/tests/test_storage.py | 18 +++++++------- 12 files changed, 51 insertions(+), 42 deletions(-) diff --git a/azure_monitor/examples/metrics/observer.py b/azure_monitor/examples/metrics/observer.py index 4f9344f..705178e 100644 --- a/azure_monitor/examples/metrics/observer.py +++ b/azure_monitor/examples/metrics/observer.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import psutil - from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export.controller import PushController @@ -21,6 +20,7 @@ def get_cpu_usage_callback(observer): labels = {"cpu_number": str(number)} observer.observe(percent, labels) + meter.register_observer( callback=get_cpu_usage_callback, name="cpu_percent", @@ -35,6 +35,7 @@ def get_ram_usage_callback(observer): ram_percent = psutil.virtual_memory().percent observer.observe(ram_percent, {}) + meter.register_observer( callback=get_ram_usage_callback, name="ram_percent", diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py index aade45d..949c015 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. # from typing import Dict + from opentelemetry.metrics import Meter from azure_monitor.sdk.auto_collection.dependency_metrics import ( diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py index 476a70e..5600612 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py @@ -2,9 +2,9 @@ # Licensed under the MIT License. import threading import time +from typing import Dict import requests -from typing import Dict from opentelemetry import context from opentelemetry.metrics import Meter diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py index 2394727..9e1927a 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import logging +from typing import Dict import psutil -from typing import Dict from opentelemetry.metrics import Meter logger = logging.getLogger(__name__) @@ -84,9 +84,7 @@ def _track_process_cpu(self, observer) -> None: # CPU cores, the returned value of cpu_percent() can be > 100.0. We # normalize the cpu process using the number of logical CPUs cpu_count = psutil.cpu_count(logical=True) - observer.observe( - PROCESS.cpu_percent() / cpu_count, self._labels - ) + observer.observe(PROCESS.cpu_percent() / cpu_count, self._labels) except Exception: # pylint: disable=broad-except logger.exception("Error handling get process cpu usage.") diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py index 4c31cac..8647ec6 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py @@ -107,9 +107,7 @@ def _track_request_duration(self, observer) -> None: except ZeroDivisionError: # If interval_count is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe( - int(last_average_duration * 1000.0), self._labels - ) + observer.observe(int(last_average_duration * 1000.0), self._labels) def _track_request_rate(self, observer) -> None: """ Track Request execution rate diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index d01c1d8..40bc286 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -195,8 +195,9 @@ def _check_storage_size(self): try: size += os.path.getsize(fp) except OSError: - logger.error("Path %s does not exist or is " - "inaccessible.", fp) + logger.error( + "Path %s does not exist or is " "inaccessible.", fp + ) continue if size >= self.max_size: logger.warning( @@ -204,7 +205,7 @@ def _check_storage_size(self): "reached. Currently at %fKB. Telemetry will be " "lost. Please consider increasing the value of " "'storage_max_size' in exporter config.", - format(size/1024) + format(size / 1024), ) return False return True diff --git a/azure_monitor/tests/auto_collection/test_auto_collection.py b/azure_monitor/tests/auto_collection/test_auto_collection.py index 884b00a..c965d4b 100644 --- a/azure_monitor/tests/auto_collection/test_auto_collection.py +++ b/azure_monitor/tests/auto_collection/test_auto_collection.py @@ -40,12 +40,8 @@ def test_constructor( self.assertEqual(mock_dependencies.called, True) self.assertEqual(mock_requests.called, True) self.assertEqual(mock_performance.call_args[0][0], self._meter) - self.assertEqual( - mock_performance.call_args[0][1], self._test_labels - ) + self.assertEqual(mock_performance.call_args[0][1], self._test_labels) self.assertEqual(mock_dependencies.call_args[0][0], self._meter) - self.assertEqual( - mock_dependencies.call_args[0][1], self._test_labels - ) + self.assertEqual(mock_dependencies.call_args[0][1], self._test_labels) self.assertEqual(mock_requests.call_args[0][0], self._meter) self.assertEqual(mock_requests.call_args[0][1], self._test_labels) diff --git a/azure_monitor/tests/auto_collection/test_dependency_metrics.py b/azure_monitor/tests/auto_collection/test_dependency_metrics.py index 8514528..df0a8a1 100644 --- a/azure_monitor/tests/auto_collection/test_dependency_metrics.py +++ b/azure_monitor/tests/auto_collection/test_dependency_metrics.py @@ -67,7 +67,9 @@ def test_track_dependency_rate(self, time_mock): dependency_metrics.dependency_map["last_time"] = 98 dependency_metrics.dependency_map["count"] = 4 metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 2) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 2 + ) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate_time_none(self, time_mock): @@ -85,7 +87,9 @@ def test_track_dependency_rate_time_none(self, time_mock): meter=self._meter, ) metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0 + ) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate_error(self, time_mock): @@ -104,7 +108,9 @@ def test_track_dependency_rate_error(self, time_mock): meter=self._meter, ) metrics_collector._track_dependency_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 5) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 5 + ) @mock.patch( "azure_monitor.sdk.auto_collection.dependency_metrics.ORIGINAL_REQUEST" diff --git a/azure_monitor/tests/auto_collection/test_performance_metrics.py b/azure_monitor/tests/auto_collection/test_performance_metrics.py index a0df358..ec81d2a 100644 --- a/azure_monitor/tests/auto_collection/test_performance_metrics.py +++ b/azure_monitor/tests/auto_collection/test_performance_metrics.py @@ -108,7 +108,9 @@ def test_track_memory(self, psutil_mock): meter=self._meter, ) performance_metrics_collector._track_memory(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 100) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 100 + ) @mock.patch("azure_monitor.sdk.auto_collection.performance_metrics.psutil") def test_track_process_cpu(self, psutil_mock): diff --git a/azure_monitor/tests/auto_collection/test_request_metrics.py b/azure_monitor/tests/auto_collection/test_request_metrics.py index 903e150..62b5a88 100644 --- a/azure_monitor/tests/auto_collection/test_request_metrics.py +++ b/azure_monitor/tests/auto_collection/test_request_metrics.py @@ -38,9 +38,7 @@ def test_constructor(self): meter=mock_meter, labels=self._test_labels ) self.assertEqual(request_metrics_collector._meter, mock_meter) - self.assertEqual( - request_metrics_collector._labels, self._test_labels - ) + self.assertEqual(request_metrics_collector._labels, self._test_labels) self.assertEqual(mock_meter.register_observer.call_count, 2) @@ -78,7 +76,9 @@ def test_track_request_duration(self): meter=self._meter, ) request_metrics_collector._track_request_duration(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 20) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 20 + ) def test_track_request_duration_error(self): request_metrics_collector = request_metrics.RequestMetrics( @@ -96,7 +96,9 @@ def test_track_request_duration_error(self): meter=self._meter, ) request_metrics_collector._track_request_duration(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0 + ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate(self, time_mock): @@ -115,7 +117,9 @@ def test_track_request_rate(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 2) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 2 + ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_time_none(self, time_mock): @@ -133,7 +137,9 @@ def test_track_request_rate_time_none(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 0) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0 + ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_error(self, time_mock): @@ -152,7 +158,9 @@ def test_track_request_rate_error(self, time_mock): meter=self._meter, ) request_metrics_collector._track_request_rate(obs) - self.assertEqual(obs.aggregators[tuple(self._test_labels.items())].current, 5) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 5 + ) def test_request_patch(self): map = request_metrics.requests_map # pylint: disable=redefined-builtin diff --git a/azure_monitor/tests/metrics/test_metrics.py b/azure_monitor/tests/metrics/test_metrics.py index e2a7baa..ee5e28e 100644 --- a/azure_monitor/tests/metrics/test_metrics.py +++ b/azure_monitor/tests/metrics/test_metrics.py @@ -145,9 +145,7 @@ def test_metric_to_envelope(self): aggregator = CounterAggregator() aggregator.update(123) aggregator.take_checkpoint() - record = MetricRecord( - aggregator, self._test_labels, self._test_metric - ) + record = MetricRecord(aggregator, self._test_labels, self._test_metric) exporter = self._exporter envelope = exporter._metric_to_envelope(record) self.assertIsInstance(envelope, Envelope) diff --git a/azure_monitor/tests/test_storage.py b/azure_monitor/tests/test_storage.py index c4258b0..cee8cb6 100644 --- a/azure_monitor/tests/test_storage.py +++ b/azure_monitor/tests/test_storage.py @@ -110,7 +110,7 @@ def test_put(self): def test_put_max_size(self): test_input = (1, 2, 3) - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd')) as stor: + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd")) as stor: size_mock = mock.Mock() size_mock.return_value = False stor._check_storage_size = size_mock @@ -119,34 +119,34 @@ def test_put_max_size(self): def test_check_storage_size_full(self): test_input = (1, 2, 3) - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd2'), 1) as stor: + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd2"), 1) as stor: stor.put(test_input) self.assertFalse(stor._check_storage_size()) def test_check_storage_size_not_full(self): test_input = (1, 2, 3) - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd3"), 1000) as stor: stor.put(test_input) self.assertTrue(stor._check_storage_size()) def test_check_storage_size_no_files(self): - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd3"), 1000) as stor: self.assertTrue(stor._check_storage_size()) def test_check_storage_size_links(self): test_input = (1, 2, 3) - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd4'), 1000) as stor: + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd4"), 1000) as stor: stor.put(test_input) - with mock.patch('os.path.islink') as os_mock: + with mock.patch("os.path.islink") as os_mock: os_mock.return_value = True self.assertTrue(stor._check_storage_size()) def test_check_storage_size_error(self): test_input = (1, 2, 3) - with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor: - with mock.patch('os.path.getsize', side_effect=throw(OSError)): + with LocalFileStorage(os.path.join(TEST_FOLDER, "asd5"), 1) as stor: + with mock.patch("os.path.getsize", side_effect=throw(OSError)): stor.put(test_input) - with mock.patch('os.path.islink') as os_mock: + with mock.patch("os.path.islink") as os_mock: os_mock.return_value = True self.assertTrue(stor._check_storage_size()) From aa220e773ea35216117859f78317780edaffb316 Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 3 Apr 2020 09:47:44 -0700 Subject: [PATCH 09/39] flake --- azure_monitor/examples/metrics/observer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_monitor/examples/metrics/observer.py b/azure_monitor/examples/metrics/observer.py index 705178e..1ce8332 100644 --- a/azure_monitor/examples/metrics/observer.py +++ b/azure_monitor/examples/metrics/observer.py @@ -14,6 +14,7 @@ ) controller = PushController(meter=meter, exporter=exporter, interval=2) + # Callback to gather cpu usage def get_cpu_usage_callback(observer): for (number, percent) in enumerate(psutil.cpu_percent(percpu=True)): @@ -30,6 +31,7 @@ def get_cpu_usage_callback(observer): label_keys=("cpu_number",), ) + # Callback to gather RAM memory usage def get_ram_usage_callback(observer): ram_percent = psutil.virtual_memory().percent From c53b5b52e2fcb1495404acc67ddf79075e9717cf Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 3 Apr 2020 09:52:58 -0700 Subject: [PATCH 10/39] pylint --- azure_monitor/src/azure_monitor/storage.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index 40bc286..063a951 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -187,16 +187,17 @@ def put(self, data, lease_period=0, silent=False): def _check_storage_size(self): size = 0 + # pylint: disable=unused-variable for dirpath, dirnames, filenames in os.walk(self.path): - for f in filenames: - fp = os.path.join(dirpath, f) + for filename in filenames: + path = os.path.join(dirpath, filename) # skip if it is symbolic link - if not os.path.islink(fp): + if not os.path.islink(path): try: - size += os.path.getsize(fp) + size += os.path.getsize(path) except OSError: logger.error( - "Path %s does not exist or is " "inaccessible.", fp + "Path %s does not exist or is " "inaccessible.", path ) continue if size >= self.max_size: From d0d194c0a5c662cc793ec550bf643c251eafc451 Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 3 Apr 2020 10:01:22 -0700 Subject: [PATCH 11/39] black --- azure_monitor/src/azure_monitor/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index 063a951..5c554cb 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -197,7 +197,8 @@ def _check_storage_size(self): size += os.path.getsize(path) except OSError: logger.error( - "Path %s does not exist or is " "inaccessible.", path + "Path %s does not exist or is " "inaccessible.", + path, ) continue if size >= self.max_size: From dcf5094b8b79addce4a22ed31c6e932f20b12693 Mon Sep 17 00:00:00 2001 From: Leighton Date: Sun, 5 Apr 2020 20:21:33 -0700 Subject: [PATCH 12/39] readme --- README.md | 158 ++++++++++++++++++++++++++++++++++++++++ azure_monitor/README.md | 150 -------------------------------------- 2 files changed, 158 insertions(+), 150 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..68244cd --- /dev/null +++ b/README.md @@ -0,0 +1,158 @@ +# OpenTelemetry Azure Monitor + +[![Gitter chat](https://img.shields.io/gitter/room/Microsoft/azure-monitor-python)](https://gitter.im/Microsoft/azure-monitor-python) +[![Build status](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python.svg?branch=master)](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python) + +## Installation + +```sh +pip install opentelemetry-azure-monitor +``` + +## Documentation + +The online documentation is available at https://opentelemetry-azure-monitor-python.readthedocs.io/. + +## Usage + + +### Trace + +The **Azure Monitor Trace Exporter** allows you to export [OpenTelemetry](https://opentelemetry.io/) traces to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). + +This example shows how to send a span "hello" to Azure Monitor. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_tracer_provider(TracerProvider()) + +# We tell OpenTelemetry who it is that is creating spans. In this case, we have +# no real name (no setup.py), so we make one up. If we had a version, we would +# also specify it here. +tracer = trace.get_tracer(__name__) + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=', +) + +# SpanExporter receives the spans and send them to the target location. +span_processor = BatchExportSpanProcessor(exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +with tracer.start_as_current_span('hello'): + print('Hello World!') +``` + +#### Integrations + +OpenTelemetry also supports several [integrations](https://github.com/open-telemetry/opentelemetry-python/tree/master/ext) which allows to integrate with third party libraries. + +This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/)_ library. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Install the `requests` integration package using ``pip install opentelemetry-ext-http-requests``. +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +import requests + +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_tracer_provider(TracerProvider()) +tracer_provider = trace.get_tracer_provider() + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=', + ) +span_processor = BatchExportSpanProcessor(exporter) +tracer_provider.add_span_processor(span_processor) + +http_requests.enable(tracer_provider) +response = requests.get(url="https://azure.microsoft.com/") +``` + +#### Modifying Traces + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an [envelope](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py#L80) data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py). +* The `AzureMonitorSpanExporter` handles `Data` data types. + +```python +from azure_monitor import AzureMonitorSpanExporter +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +# Callback function to add os_type: linux to span properties +def callback_function(envelope): + envelope.data.baseData.properties['os_type'] = 'linux' + return True + +exporter = AzureMonitorSpanExporter( + connection_string='InstrumentationKey=' +) +exporter.add_telemetry_processor(callback_function) + +trace.set_tracer_provider(TracerProvider()) +tracer = trace.get_tracer(__name__) +span_processor = BatchExportSpanProcessor(exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +with tracer.start_as_current_span('hello'): + print('Hello World!') +``` + +### Metrics + +The **Azure Monitor Metrics Exporter** allows you to export metrics to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). + +This example shows how to track a counter metric and send it as telemetry every export interval. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + +```python +import time + +from azure_monitor import AzureMonitorMetricsExporter +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export.controller import PushController + +metrics.set_meter_provider(MeterProvider()) +meter = metrics.get_meter(__name__) +exporter = AzureMonitorMetricsExporter( + connection_string='InstrumentationKey=' +) +controller = PushController(meter, exporter, 5) + +requests_counter = meter.create_metric( + name="requests", + description="number of requests", + unit="1", + value_type=int, + metric_type=Counter, + label_keys=("environment",), +) + +testing_labels = {"environment": "testing"} + +requests_counter.add(25, testing_labels) +time.sleep(100) +``` diff --git a/azure_monitor/README.md b/azure_monitor/README.md index e31d400..98b692d 100644 --- a/azure_monitor/README.md +++ b/azure_monitor/README.md @@ -1,7 +1,5 @@ # OpenTelemetry Azure Monitor SDKs and Exporters -[![Gitter chat](https://img.shields.io/gitter/room/Microsoft/azure-monitor-python)](https://gitter.im/Microsoft/azure-monitor-python) -[![Build status](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python.svg?branch=master)](https://travis-ci.org/microsoft/opentelemetry-azure-monitor-python) [![PyPI version](https://badge.fury.io/py/opentelemetry-azure-monitor.svg)](https://badge.fury.io/py/opentelemetry-azure-monitor) ## Installation @@ -10,154 +8,6 @@ pip install opentelemetry-azure-monitor ``` -## Documentation - -The online documentation is available at https://opentelemetry-azure-monitor-python.readthedocs.io/. - - -## Usage - -### Trace - -The **Azure Monitor Trace Exporter** allows you to export [OpenTelemetry](https://opentelemetry.io/) traces to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). - -This example shows how to send a span "hello" to Azure Monitor. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -trace.set_tracer_provider(TracerProvider()) - -# We tell OpenTelemetry who it is that is creating spans. In this case, we have -# no real name (no setup.py), so we make one up. If we had a version, we would -# also specify it here. -tracer = trace.get_tracer(__name__) - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=', -) - -# SpanExporter receives the spans and send them to the target location. -span_processor = BatchExportSpanProcessor(exporter) -trace.get_tracer_provider().add_span_processor(span_processor) - -with tracer.start_as_current_span('hello'): - print('Hello World!') -``` - -#### Integrations - -OpenTelemetry also supports several [integrations](https://github.com/open-telemetry/opentelemetry-python/tree/master/ext) which allows to integrate with third party libraries. - -This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/)_ library. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Install the `requests` integration package using ``pip install opentelemetry-ext-http-requests``. -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -import requests - -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -trace.set_tracer_provider(TracerProvider()) -tracer_provider = trace.get_tracer_provider() - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=', - ) -span_processor = BatchExportSpanProcessor(exporter) -tracer_provider.add_span_processor(span_processor) - -http_requests.enable(tracer_provider) -response = requests.get(url="https://azure.microsoft.com/") -``` - -#### Modifying Traces - -* You can pass a callback function to the exporter to process telemetry before it is exported. -* Your callback function can return `False` if you do not want this envelope exported. -* Your callback function must accept an [envelope](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py#L80) data type as its parameter. -* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/microsoft/opentelemetry-exporters-python/blob/master/azure_monitor/src/azure_monitor/protocol.py). -* The `AzureMonitorSpanExporter` handles `Data` data types. - -```python -from azure_monitor import AzureMonitorSpanExporter -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -# Callback function to add os_type: linux to span properties -def callback_function(envelope): - envelope.data.baseData.properties['os_type'] = 'linux' - return True - -exporter = AzureMonitorSpanExporter( - connection_string='InstrumentationKey=' -) -exporter.add_telemetry_processor(callback_function) - -trace.set_tracer_provider(TracerProvider()) -tracer = trace.get_tracer(__name__) -span_processor = BatchExportSpanProcessor(exporter) -trace.get_tracer_provider().add_span_processor(span_processor) - -with tracer.start_as_current_span('hello'): - print('Hello World!') -``` - -### Metrics - -The **Azure Monitor Metrics Exporter** allows you to export metrics to [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/). - -This example shows how to track a counter metric and send it as telemetry every export interval. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found [here](https://docs.microsoft.com/azure/azure-monitor/app/create-new-resource). -* Place your instrumentation key in a `connection string` and directly into your code. -* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. - -```python -import time - -from azure_monitor import AzureMonitorMetricsExporter -from opentelemetry import metrics -from opentelemetry.sdk.metrics import Counter, MeterProvider -from opentelemetry.sdk.metrics.export.controller import PushController - -metrics.set_meter_provider(MeterProvider()) -meter = metrics.get_meter(__name__) -exporter = AzureMonitorMetricsExporter( - connection_string='InstrumentationKey=' -) -controller = PushController(meter, exporter, 5) - -requests_counter = meter.create_metric( - name="requests", - description="number of requests", - unit="1", - value_type=int, - metric_type=Counter, - label_keys=("environment",), -) - -testing_labels = {"environment": "testing"} - -requests_counter.add(25, testing_labels) -time.sleep(100) -``` - # References [Azure Monitor](https://docs.microsoft.com/azure/azure-monitor/) From 60fe4a4a2498f7829f58c6794b237c0da011e357 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Wed, 15 Apr 2020 16:51:00 -0700 Subject: [PATCH 13/39] Adding live metrics capabilities --- .../auto_collection/live_metrics/__init__.py | 151 ++++++++++++++++ .../auto_collection/live_metrics/exporter.py | 163 ++++++++++++++++++ .../auto_collection/live_metrics/manager.py | 148 ++++++++++++++++ .../auto_collection/live_metrics/sender.py | 47 +++++ .../auto_collection/live_metrics/__init__.py | 2 + .../live_metrics/test_exporter.py | 81 +++++++++ .../live_metrics/test_manager.py | 39 +++++ .../live_metrics/test_sender.py | 85 +++++++++ 8 files changed, 716 insertions(+) create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/__init__.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_exporter.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_manager.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_sender.py diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py new file mode 100644 index 0000000..a8e7887 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import time +import typing +import uuid + +from azure_monitor.protocol import BaseObject +from azure_monitor.utils import azure_monitor_context + +LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" +STREAM_ID = str(uuid.uuid4()) + + +def create_metric_envelope(instrumentation_key: str): + envelope = LiveMetricEnvelope( + documents=None, + instance=azure_monitor_context.get("ai.cloud.roleInstance"), + instrumentation_key=instrumentation_key, + invariant_version=1, # 1 -> v1 QPS protocol, + machine_name=azure_monitor_context.get("ai.device.id"), + metrics=None, + stream_id=STREAM_ID, + timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), + version=azure_monitor_context.get("ai.internal.sdkVersion"), + ) + return envelope + + +class LiveMetricDocumentProperty(BaseObject): + + __slots__ = ("key", "value") + + def __init__(self, key: str, value: str) -> None: + self.key = key + self.value = value + + +class LiveMetricDocument(BaseObject): + + __slots__ = ( + "__type", + "document_type", + "version", + "operation_id", + "properties", + ) + + def __init__( + self, + __type: str = "", + document_type: str = "", + version: str = "", + operation_id: str = "", + properties: typing.List[LiveMetricDocumentProperty] = None, + ) -> None: + self.__type = __type + self.document_type = document_type + self.version = version + self.operation_id = operation_id + self.properties = properties + + def to_dict(self): + return { + "__type": self.__type, + "DocumentType": self.document_type, + "Version": self.version, + "OperationId": self.operation_id, + "Properties": self.properties.to_dict() + if self.properties + else None, + } + + +class LiveMetric(BaseObject): + + __slots__ = ("name", "value", "weight") + + def __init__(self, name: str, value: str, weight: int) -> None: + self.name = name + self.value = value + self.weight = weight + + def to_dict(self): + return {"Name": self.name, "Value": self.value, "Weight": self.weight} + + +class LiveMetricEnvelope(BaseObject): + """Envelope to send data to Live Metrics service. + + Args: + documents: An array of detailed failure documents, each representing a full SDK telemetry item. + instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. + instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, + it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. + invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. + machine_name: Machine name. + metrics: Metrics + stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. + timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". + version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). + """ + + __slots__ = ( + "documents", + "instance", + "instrumentation_key", + "invariant_version", + "machine_name", + "metrics", + "stream_id", + "timestamp", + "version", + ) + + def __init__( + self, + documents: typing.List[LiveMetricDocument] = None, + instance: str = "", + instrumentation_key: str = "", + invariant_version: int = 1, + machine_name: str = "", + metrics: typing.List[LiveMetric] = None, + stream_id: str = "", + timestamp: str = "", + version: str = "", + ) -> None: + if metrics is None: + metrics = [] + self.documents = documents + self.instance = instance + self.instrumentation_key = instrumentation_key + self.invariant_version = invariant_version + self.machine_name = machine_name + self.metrics = metrics + self.stream_id = stream_id + self.timestamp = timestamp + self.version = version + + def to_dict(self): + return { + "Documents": self.documents.to_dict() if self.documents else None, + "Instance": self.instance, + "InstrumentationKey": self.instrumentation_key, + "InvariantVersion": self.invariant_version, + "MachineName": self.machine_name, + "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), + "StreamId": self.stream_id, + "Timestamp": self.timestamp, + "Version": self.version, + } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py new file mode 100644 index 0000000..3f7674f --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -0,0 +1,163 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import collections +import logging +import typing + +from opentelemetry.sdk.metrics import Counter, Observer +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExporter, + MetricsExportResult, +) + +from azure_monitor.protocol import Envelope +from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import ( + LiveMetric, + LiveMetricDocument, + LiveMetricDocumentProperty, + LiveMetricEnvelope, +) +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + +logger = logging.getLogger(__name__) + + +class LiveMetricsExporter(MetricsExporter): + def __init__(self, instrumentation_key): + self._instrumentation_key = instrumentation_key + self._sender = LiveMetricsSender(self._instrumentation_key) + self._subscribed = True + self._document_envelopes = collections.deque() + + def add_document(self, envelope: Envelope): + self._document_envelopes.append(envelope) + + def export( + self, metric_records: typing.Sequence[MetricRecord] + ) -> MetricsExportResult: + envelope = self._metric_to_live_metrics_envelope(metric_records) + try: + response = self._sender.post(envelope) + if response.ok: + self._subscribed = ( + response.headers.get( + live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER + ) + == "true" + ) + return MetricsExportResult.SUCCESS + + except Exception: # pylint: disable=broad-except + logger.exception("Exception occurred while exporting the data.") + + return MetricsExportResult.FAILED_NOT_RETRYABLE + + def _metric_to_live_metrics_envelope( + self, metric_records: typing.Sequence[MetricRecord] + ) -> LiveMetricEnvelope: + + envelope = live_metrics.create_metric_envelope( + self._instrumentation_key + ) + envelope.documents = self._get_live_metric_documents() + envelope.metrics = [] + + # Add metrics + for metric_record in metric_records: + value = 0 + metric = metric_record.metric + if isinstance(metric, Counter): + value = metric_record.aggregator.checkpoint + elif isinstance(metric, Observer): + value = metric_record.aggregator.checkpoint.last + if not value: + value = 0 + envelope.metrics.append( + LiveMetric(name=metric.name, value=value, weight=1) + ) + + return envelope + + def _get_live_metric_documents( + self, + ) -> typing.Sequence[LiveMetricDocument]: + live_metric_documents = [] + while self._document_envelopes: + for envelope in self._document_envelopes.popleft(): + base_type = envelope.data.baseType + if base_type: + document = LiveMetricDocument( + __type=self._get_live_metric_type(base_type), + document_type=self._get_live_metric_document_type( + base_type + ), + properties=self._get_aggregated_properties(envelope), + version="1.0", + ) + live_metric_documents.append(document) + else: + logger.warning( + "Document type invalid; not sending live metric document" + ) + + return live_metric_documents + + def _get_live_metric_type(self, base_type: str) -> str: + if base_type == "EventData": + return "EventTelemetryDocument" + elif base_type == "ExceptionData": + return "ExceptionTelemetryDocument" + elif base_type == "MessageData": + return "TraceTelemetryDocument" + elif base_type == "MetricData": + return "MetricTelemetryDocument" + elif base_type == "RequestData": + return "RequestTelemetryDocument" + elif base_type == "RemoteDependencyData": + return "DependencyTelemetryDocument" + elif base_type == "AvailabilityData": + return "AvailabilityTelemetryDocument" + + def _get_live_metric_document_type(self, base_type: str) -> str: + if base_type == "EventData": + return "Event" + elif base_type == "ExceptionData": + return "Exception" + elif base_type == "MessageData": + return "Trace" + elif base_type == "MetricData": + return "Metric" + elif base_type == "RequestData": + return "Request" + elif base_type == "RemoteDependencyData": + return "RemoteDependency" + elif base_type == "AvailabilityData": + return "Availability" + + def _get_aggregated_properties( + self, envelope: Envelope + ) -> typing.List[LiveMetricDocumentProperty]: + + aggregated_properties = [] + measurements = ( + envelope.data.base_data.measurements + if envelope.data.base_data.measurements + else [] + ) + for key in measurements: + prop = LiveMetricDocumentProperty(key=key, value=measurements[key]) + aggregated_properties.append(prop) + properties = ( + envelope.data.base_data.properties + if envelope.data.base_data.properties + else [] + ) + for key in properties: + prop = LiveMetricDocumentProperty(key=key, value=properties[key]) + aggregated_properties.append(prop) + return aggregated_properties diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py new file mode 100644 index 0000000..23cb419 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import threading +import time + +from opentelemetry.sdk.metrics.export import MetricsExportResult + +from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + +FALLBACK_INTERVAL = 60 +PING_INTERVAL = 5 +POST_INTERVAL = 1 +MAIN_INTERVAL = 2 + + +class LiveMetricsManager(threading.Thread): + + daemon = True + + def __init__(self, meter, instrumentation_key): + super().__init__() + self.thread_event = threading.Event() + self.interval = MAIN_INTERVAL + self._instrumentation_key = instrumentation_key + self._is_user_subscribed = False + self._meter = meter + self._post = None + self._ping = LiveMetricsPing(self._instrumentation_key) + self.start() + + def run(self): + self.check_if_user_is_subscribed() + while not self.thread_event.wait(self.interval): + self.check_if_user_is_subscribed() + + def check_if_user_is_subscribed(self): + if self._ping: + if self._ping.is_user_subscribed: + # Switch to Post + self._ping.shutdown() + self._ping = None + self._post = LiveMetricsPost( + self._meter, self._instrumentation_key + ) + if self._post: + if not self._post.is_user_subscribed: + # Switch to Ping + self._post.shutdown() + self._post = None + self._ping = LiveMetricsPing(self._instrumentation_key) + + def shutdown(self): + self.thread_event.set() + + +class LiveMetricsPing(threading.Thread): + + daemon = True + + def __init__(self, instrumentation_key): + super().__init__() + self.instrumentation_key = instrumentation_key + self.thread_event = threading.Event() + self.interval = PING_INTERVAL + self.is_user_subscribed = False + self.last_send_succeeded = False + self.last_request_success_time = 0 + self.sender = LiveMetricsSender(self.instrumentation_key) + self.start() + + def run(self): + self.ping() + while not self.thread_event.wait(self.interval): + self.ping() + + def ping(self): + envelope = live_metrics.create_metric_envelope( + self.instrumentation_key + ) + response = self.sender.ping(envelope) + if response.ok: + if not self.last_send_succeeded: + self.interval = PING_INTERVAL + self.last_send_succeeded = True + self.last_request_success_time = time.time() + if ( + response.headers.get( + live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER + ) + == "true" + ): + self.is_user_subscribed = True + else: + self.last_send_succeeded = False + if time.time() >= self.last_request_success_time + 20: + self.interval = FALLBACK_INTERVAL + + def shutdown(self): + self.thread_event.set() + + +class LiveMetricsPost(threading.Thread): + + daemon = True + + def __init__(self, meter, instrumentation_key): + super().__init__() + self.instrumentation_key = instrumentation_key + self.meter = meter + self.thread_event = threading.Event() + self.interval = POST_INTERVAL + self.is_user_subscribed = True + self.last_send_succeeded = False + self.last_request_success_time = time.time() + self.exporter = LiveMetricsExporter(self.instrumentation_key) + self.start() + + def run(self): + self.post() + while not self.thread_event.wait(self.interval): + self.post() + + def post(self): + self.meter.collect() + result = self.exporter.export(self.meter.batcher.checkpoint_set()) + self.meter.batcher.finished_collection() + if result == MetricsExportResult.SUCCESS: + self.last_request_success_time = time.time() + if not self.last_send_succeeded: + self.interval = POST_INTERVAL + self.last_send_succeeded = True + + if self.exporter.subscribed: + self.is_user_subscribed = True + else: + self.last_send_succeeded = False + if time.time() >= self.last_request_success_time + 60: + self.interval = FALLBACK_INTERVAL + + def shutdown(self): + self.thread_event.set() diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py new file mode 100644 index 0000000..04df2d8 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import json +import logging +import time + +import requests + +from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope + +DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" +LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" + +logger = logging.getLogger(__name__) + + +class LiveMetricsSender: + def __init__(self, instrumentation_key: str): + self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT + self._instrumentation_key = instrumentation_key + + def ping(self, envelope: LiveMetricEnvelope): + return self._send_request(json.dumps(envelope.to_dict()), "ping") + + def post(self, envelope: LiveMetricEnvelope): + return self._send_request(json.dumps([envelope.to_dict()]), "post") + + def _send_request(self, data: str, request_type: str) -> requests.Response: + try: + url = "{0}/QuickPulseService.svc/{1}?ikey={2}".format( + self._endpoint, request_type, self._instrumentation_key + ) + response = requests.post( + url=url, + data=data, + headers={ + "Expect": "100-continue", + "Content-Type": "application/json; charset=utf-8", + LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( + round(time.time()) * 1000 + ), + }, + ) + except Exception as ex: + logger.warning("Failed to send live metrics: %s.", ex) + return response diff --git a/azure_monitor/tests/auto_collection/live_metrics/__init__.py b/azure_monitor/tests/auto_collection/live_metrics/__init__.py new file mode 100644 index 0000000..5b7f7a9 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py new file mode 100644 index 0000000..613f5e0 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from unittest import mock + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult +from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator +import requests + +from azure_monitor.protocol import Envelope +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) + + +# pylint: disable=protected-access +class TestLiveMetricsExporter(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + metrics.set_meter_provider(MeterProvider()) + cls._meter = metrics.get_meter(__name__) + cls._test_metric = cls._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + cls._test_labels = tuple({"environment": "staging"}.items()) + + def test_constructor(self): + """Test the constructor.""" + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + self.assertEqual(exporter._subscribed, True) + self.assertEqual( + exporter._instrumentation_key, self._instrumentation_key + ) + + def test_add_document(self): + """Test adding a document.""" + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + envelope = Envelope() + exporter.add_document(envelope) + self.assertEqual(exporter._document_envelopes.pop(), envelope) + + def test_export(self): + """Test export.""" + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" + ) as request: + response = requests.Response() + response.status_code = 200 + request.return_value = response + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.SUCCESS) + + def test_export_failed(self): + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" + ) as request: + response = requests.Response() + response.status_code = 400 + request.return_value = response + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py new file mode 100644 index 0000000..f0291a4 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import time +import unittest +from unittest import mock + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider + +from azure_monitor.sdk.auto_collection.live_metrics.manager import ( + LiveMetricsManager, +) + + +# pylint: disable=protected-access +class TestAutoCollection(unittest.TestCase): + @classmethod + def setUpClass(cls): + metrics.set_meter_provider(MeterProvider()) + cls._meter = metrics.get_meter(__name__) + cls._test_metric = cls._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + testing_labels = {"environment": "testing"} + cls._test_metric.add(5, testing_labels) + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + + @classmethod + def tearDownClass(cls): + metrics._METER_PROVIDER = None + + def test_constructor(self): + """Test the constructor.""" + # LiveMetricsManager( + # meter=self._meter, instrumentation_key=self._instrumentation_key + # ) + # time.sleep(200) + # self.assertEqual(True, False) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py new file mode 100644 index 0000000..5fdead2 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from unittest import mock + +from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + + +# pylint: disable=protected-access +class TestLiveMetricsSender(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + + def test_constructor(self): + """Test the constructor.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + self.assertEqual( + sender._endpoint, "https://rt.services.visualstudio.com" + ) + self.assertEqual( + sender._instrumentation_key, self._instrumentation_key + ) + + def test_ping(self): + """Test ping.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + envelope = LiveMetricEnvelope() + with mock.patch("requests.post") as request: + sender.ping(envelope) + self.assertTrue(request.called) + self.assertEqual( + request.call_args[1].get("url"), + "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey={0}".format( + self._instrumentation_key + ), + ) + self.assertEqual( + request.call_args[1].get("data"), + '{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}', + ) + headers = request.call_args[1].get("headers") + self.assertEqual(headers.get("Expect"), "100-continue") + self.assertEqual( + headers.get("Content-Type"), "application/json; charset=utf-8" + ) + self.assertTrue( + headers.get("x-ms-qps-transmission-time").isdigit() + ) + + def test_post(self): + """Test post.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + envelope = LiveMetricEnvelope() + with mock.patch("requests.post") as request: + sender.post(envelope) + self.assertTrue(request.called) + self.assertEqual( + request.call_args[1].get("url"), + "https://rt.services.visualstudio.com/QuickPulseService.svc/post?ikey={0}".format( + self._instrumentation_key + ), + ) + self.assertEqual( + request.call_args[1].get("data"), + '[{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}]', + ) + headers = request.call_args[1].get("headers") + self.assertEqual(headers.get("Expect"), "100-continue") + self.assertEqual( + headers.get("Content-Type"), "application/json; charset=utf-8" + ) + self.assertTrue( + headers.get("x-ms-qps-transmission-time").isdigit() + ) From c18e58932bc09aaf71f6064a24dea40136322cfa Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Thu, 16 Apr 2020 14:55:52 -0700 Subject: [PATCH 14/39] Fix shutdown --- .../auto_collection/live_metrics/manager.py | 4 ++++ .../live_metrics/test_manager.py | 22 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 23cb419..f7f2f6e 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -57,6 +57,10 @@ def check_if_user_is_subscribed(self): self._ping = LiveMetricsPing(self._instrumentation_key) def shutdown(self): + if self._ping: + self._ping.shutdown() + if self._post: + self._post.shutdown() self.thread_event.set() diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index f0291a4..966fac2 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -14,7 +14,7 @@ # pylint: disable=protected-access -class TestAutoCollection(unittest.TestCase): +class TestLiveMetricsManager(unittest.TestCase): @classmethod def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) @@ -25,15 +25,25 @@ def setUpClass(cls): testing_labels = {"environment": "testing"} cls._test_metric.add(5, testing_labels) cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + cls._manager = None @classmethod def tearDownClass(cls): metrics._METER_PROVIDER = None + def tearDown(self): + self._manager.shutdown() + def test_constructor(self): """Test the constructor.""" - # LiveMetricsManager( - # meter=self._meter, instrumentation_key=self._instrumentation_key - # ) - # time.sleep(200) - # self.assertEqual(True, False) + with mock.patch("requests.post"): + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self.assertFalse(self._manager._is_user_subscribed) + self.assertEqual( + self._manager._instrumentation_key, self._instrumentation_key + ) + self.assertEqual(self._manager._meter, self._meter) + From 09a623f0dbe7286584d9ab10d8a56082f336ac6e Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 21 Apr 2020 15:53:07 -0700 Subject: [PATCH 15/39] Addressing comments --- azure_monitor/src/azure_monitor/protocol.py | 124 +++++++++++++++++ .../auto_collection/live_metrics/__init__.py | 126 +----------------- .../auto_collection/live_metrics/exporter.py | 11 +- .../auto_collection/live_metrics/manager.py | 18 +++ .../auto_collection/live_metrics/sender.py | 7 +- .../live_metrics/test_exporter.py | 2 +- .../live_metrics/test_manager.py | 32 +++++ 7 files changed, 190 insertions(+), 130 deletions(-) diff --git a/azure_monitor/src/azure_monitor/protocol.py b/azure_monitor/src/azure_monitor/protocol.py index 9f68f16..e10c8d4 100644 --- a/azure_monitor/src/azure_monitor/protocol.py +++ b/azure_monitor/src/azure_monitor/protocol.py @@ -561,3 +561,127 @@ def to_dict(self): "properties": self.properties, "measurements": self.measurements, } + + +class LiveMetricDocumentProperty(BaseObject): + + __slots__ = ("key", "value") + + def __init__(self, key: str, value: str) -> None: + self.key = key + self.value = value + + +class LiveMetricDocument(BaseObject): + + __slots__ = ( + "__type", + "document_type", + "version", + "operation_id", + "properties", + ) + + def __init__( + self, + __type: str = "", + document_type: str = "", + version: str = "", + operation_id: str = "", + properties: typing.List[LiveMetricDocumentProperty] = None, + ) -> None: + self.__type = __type + self.document_type = document_type + self.version = version + self.operation_id = operation_id + self.properties = properties + + def to_dict(self): + return { + "__type": self.__type, + "DocumentType": self.document_type, + "Version": self.version, + "OperationId": self.operation_id, + "Properties": self.properties.to_dict() + if self.properties + else None, + } + + +class LiveMetric(BaseObject): + + __slots__ = ("name", "value", "weight") + + def __init__(self, name: str, value: str, weight: int) -> None: + self.name = name + self.value = value + self.weight = weight + + def to_dict(self): + return {"Name": self.name, "Value": self.value, "Weight": self.weight} + + +class LiveMetricEnvelope(BaseObject): + """Envelope to send data to Live Metrics service. + + Args: + documents: An array of detailed failure documents, each representing a full SDK telemetry item. + instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. + instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, + it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. + invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. + machine_name: Machine name. + metrics: Metrics + stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. + timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". + version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). + """ + + __slots__ = ( + "documents", + "instance", + "instrumentation_key", + "invariant_version", + "machine_name", + "metrics", + "stream_id", + "timestamp", + "version", + ) + + def __init__( + self, + documents: typing.List[LiveMetricDocument] = None, + instance: str = "", + instrumentation_key: str = "", + invariant_version: int = 1, + machine_name: str = "", + metrics: typing.List[LiveMetric] = None, + stream_id: str = "", + timestamp: str = "", + version: str = "", + ) -> None: + if metrics is None: + metrics = [] + self.documents = documents + self.instance = instance + self.instrumentation_key = instrumentation_key + self.invariant_version = invariant_version + self.machine_name = machine_name + self.metrics = metrics + self.stream_id = stream_id + self.timestamp = timestamp + self.version = version + + def to_dict(self): + return { + "Documents": self.documents.to_dict() if self.documents else None, + "Instance": self.instance, + "InstrumentationKey": self.instrumentation_key, + "InvariantVersion": self.invariant_version, + "MachineName": self.machine_name, + "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), + "StreamId": self.stream_id, + "Timestamp": self.timestamp, + "Version": self.version, + } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py index a8e7887..556060d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -5,7 +5,7 @@ import typing import uuid -from azure_monitor.protocol import BaseObject +from azure_monitor.protocol import BaseObject, LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" @@ -25,127 +25,3 @@ def create_metric_envelope(instrumentation_key: str): version=azure_monitor_context.get("ai.internal.sdkVersion"), ) return envelope - - -class LiveMetricDocumentProperty(BaseObject): - - __slots__ = ("key", "value") - - def __init__(self, key: str, value: str) -> None: - self.key = key - self.value = value - - -class LiveMetricDocument(BaseObject): - - __slots__ = ( - "__type", - "document_type", - "version", - "operation_id", - "properties", - ) - - def __init__( - self, - __type: str = "", - document_type: str = "", - version: str = "", - operation_id: str = "", - properties: typing.List[LiveMetricDocumentProperty] = None, - ) -> None: - self.__type = __type - self.document_type = document_type - self.version = version - self.operation_id = operation_id - self.properties = properties - - def to_dict(self): - return { - "__type": self.__type, - "DocumentType": self.document_type, - "Version": self.version, - "OperationId": self.operation_id, - "Properties": self.properties.to_dict() - if self.properties - else None, - } - - -class LiveMetric(BaseObject): - - __slots__ = ("name", "value", "weight") - - def __init__(self, name: str, value: str, weight: int) -> None: - self.name = name - self.value = value - self.weight = weight - - def to_dict(self): - return {"Name": self.name, "Value": self.value, "Weight": self.weight} - - -class LiveMetricEnvelope(BaseObject): - """Envelope to send data to Live Metrics service. - - Args: - documents: An array of detailed failure documents, each representing a full SDK telemetry item. - instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. - instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, - it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. - invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. - machine_name: Machine name. - metrics: Metrics - stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. - timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". - version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). - """ - - __slots__ = ( - "documents", - "instance", - "instrumentation_key", - "invariant_version", - "machine_name", - "metrics", - "stream_id", - "timestamp", - "version", - ) - - def __init__( - self, - documents: typing.List[LiveMetricDocument] = None, - instance: str = "", - instrumentation_key: str = "", - invariant_version: int = 1, - machine_name: str = "", - metrics: typing.List[LiveMetric] = None, - stream_id: str = "", - timestamp: str = "", - version: str = "", - ) -> None: - if metrics is None: - metrics = [] - self.documents = documents - self.instance = instance - self.instrumentation_key = instrumentation_key - self.invariant_version = invariant_version - self.machine_name = machine_name - self.metrics = metrics - self.stream_id = stream_id - self.timestamp = timestamp - self.version = version - - def to_dict(self): - return { - "Documents": self.documents.to_dict() if self.documents else None, - "Instance": self.instance, - "InstrumentationKey": self.instrumentation_key, - "InvariantVersion": self.invariant_version, - "MachineName": self.machine_name, - "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), - "StreamId": self.stream_id, - "Timestamp": self.timestamp, - "Version": self.version, - } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 3f7674f..1b48a79 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -14,7 +14,7 @@ from azure_monitor.protocol import Envelope from azure_monitor.sdk.auto_collection import live_metrics -from azure_monitor.sdk.auto_collection.live_metrics import ( +from azure_monitor.protocol import ( LiveMetric, LiveMetricDocument, LiveMetricDocumentProperty, @@ -28,10 +28,15 @@ class LiveMetricsExporter(MetricsExporter): + """Live Metrics Exporter + + Export data to Azure Live Metrics service and determine if user is subscribed. + """ + def __init__(self, instrumentation_key): self._instrumentation_key = instrumentation_key self._sender = LiveMetricsSender(self._instrumentation_key) - self._subscribed = True + self.subscribed = True self._document_envelopes = collections.deque() def add_document(self, envelope: Envelope): @@ -44,7 +49,7 @@ def export( try: response = self._sender.post(envelope) if response.ok: - self._subscribed = ( + self.subscribed = ( response.headers.get( live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index f7f2f6e..0b89558 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -4,6 +4,7 @@ import threading import time +from opentelemetry.context import attach, detach, set_value from opentelemetry.sdk.metrics.export import MetricsExportResult from azure_monitor.sdk.auto_collection import live_metrics @@ -21,6 +22,11 @@ class LiveMetricsManager(threading.Thread): + """Live Metrics Manager + + It will start Live Metrics process when instantiated, + responsible for switching between ping and post actions. + """ daemon = True @@ -65,6 +71,10 @@ def shutdown(self): class LiveMetricsPing(threading.Thread): + """Ping to Live Metrics service + + Ping to determine if user is subscribed and live metrics need to be send. + """ daemon = True @@ -88,7 +98,9 @@ def ping(self): envelope = live_metrics.create_metric_envelope( self.instrumentation_key ) + token = attach(set_value("suppress_instrumentation", True)) response = self.sender.ping(envelope) + detach(token) if response.ok: if not self.last_send_succeeded: self.interval = PING_INTERVAL @@ -111,6 +123,10 @@ def shutdown(self): class LiveMetricsPost(threading.Thread): + """Post to Live Metrics service + + Post to send live metrics data when user is subscribed. + """ daemon = True @@ -133,7 +149,9 @@ def run(self): def post(self): self.meter.collect() + token = attach(set_value("suppress_instrumentation", True)) result = self.exporter.export(self.meter.batcher.checkpoint_set()) + detach(token) self.meter.batcher.finished_collection() if result == MetricsExportResult.SUCCESS: self.last_request_success_time = time.time() diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index 04df2d8..960a08d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -7,7 +7,7 @@ import requests -from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" @@ -16,6 +16,11 @@ class LiveMetricsSender: + """Live Metrics Sender + + Send HTTP requests to Live Metrics service + """ + def __init__(self, instrumentation_key: str): self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT self._instrumentation_key = instrumentation_key diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index 613f5e0..d2b9a44 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -33,7 +33,7 @@ def test_constructor(self): exporter = LiveMetricsExporter( instrumentation_key=self._instrumentation_key ) - self.assertEqual(exporter._subscribed, True) + self.assertEqual(exporter.subscribed, True) self.assertEqual( exporter._instrumentation_key, self._instrumentation_key ) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 966fac2..3b5b62b 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -46,4 +46,36 @@ def test_constructor(self): self._manager._instrumentation_key, self._instrumentation_key ) self.assertEqual(self._manager._meter, self._meter) + self.assertIsNotNone(self._manager._ping) + def test_switch(self): + """Test manager switch between ping and post.""" + with mock.patch("requests.post"): + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._manager._ping.is_user_subscribed = True + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._ping) + self.assertIsNotNone(self._manager._post) + self._manager._post.is_user_subscribed = False + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._post) + self.assertIsNotNone(self._manager._ping) + + def test_ping(self): + """Test ping send requests to Live Metrics service.""" + with mock.patch("requests.post") as request: + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._manager._ping.is_user_subscribed = True + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._ping) + self.assertIsNotNone(self._manager._post) + self._manager._post.is_user_subscribed = False + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._post) + self.assertIsNotNone(self._manager._ping) From 5f1f5b6b53e22198cfe2161994ce46c7409d1e63 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 21 Apr 2020 17:31:49 -0700 Subject: [PATCH 16/39] Adding more tests --- .../auto_collection/live_metrics/exporter.py | 4 +- .../auto_collection/live_metrics/manager.py | 4 +- .../live_metrics/test_exporter.py | 2 +- .../live_metrics/test_manager.py | 113 ++++++++++++++++-- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 1b48a79..43043f7 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -12,14 +12,14 @@ MetricsExportResult, ) -from azure_monitor.protocol import Envelope -from azure_monitor.sdk.auto_collection import live_metrics from azure_monitor.protocol import ( + Envelope, LiveMetric, LiveMetricDocument, LiveMetricDocumentProperty, LiveMetricEnvelope, ) +from azure_monitor.sdk.auto_collection import live_metrics from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 0b89558..3a08308 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -159,8 +159,8 @@ def post(self): self.interval = POST_INTERVAL self.last_send_succeeded = True - if self.exporter.subscribed: - self.is_user_subscribed = True + if not self.exporter.subscribed: + self.is_user_subscribed = False else: self.last_send_succeeded = False if time.time() >= self.last_request_success_time + 60: diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index d2b9a44..987a948 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -4,11 +4,11 @@ import unittest from unittest import mock +import requests from opentelemetry import metrics from opentelemetry.sdk.metrics import Counter, MeterProvider from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator -import requests from azure_monitor.protocol import Envelope from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 3b5b62b..7630347 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -10,6 +10,8 @@ from azure_monitor.sdk.auto_collection.live_metrics.manager import ( LiveMetricsManager, + LiveMetricsPing, + LiveMetricsPost, ) @@ -26,13 +28,23 @@ def setUpClass(cls): cls._test_metric.add(5, testing_labels) cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" cls._manager = None + cls._ping = None + cls._post = None @classmethod def tearDownClass(cls): metrics._METER_PROVIDER = None def tearDown(self): - self._manager.shutdown() + if self._manager: + self._manager.shutdown() + self._manager = None + if self._ping: + self._ping.shutdown() + self._ping = None + if self._post: + self._post.shutdown() + self._post = None def test_constructor(self): """Test the constructor.""" @@ -50,12 +62,16 @@ def test_constructor(self): def test_switch(self): """Test manager switch between ping and post.""" - with mock.patch("requests.post"): + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) self._manager = LiveMetricsManager( meter=self._meter, instrumentation_key=self._instrumentation_key, ) - self._manager._ping.is_user_subscribed = True + self._manager.interval = 60 + time.sleep(1) self._manager.check_if_user_is_subscribed() self.assertIsNone(self._manager._ping) self.assertIsNotNone(self._manager._post) @@ -64,18 +80,89 @@ def test_switch(self): self.assertIsNone(self._manager._post) self.assertIsNotNone(self._manager._ping) - def test_ping(self): + def test_ping_ok(self): """Test ping send requests to Live Metrics service.""" with mock.patch("requests.post") as request: - self._manager = LiveMetricsManager( + request.return_value = MockResponse(200, None, {}) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.ping() + self.assertTrue(request.called) + self.assertTrue(self._ping.last_request_success_time > 0) + self.assertTrue(self._ping.last_send_succeeded) + self.assertFalse(self._ping.is_user_subscribed) + + def test_ping_subscribed(self): + """Test ping when user is subscribed.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.ping() + self.assertTrue(self._ping.is_user_subscribed) + + def test_ping_error(self): + """Test ping when failure.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse(400, None, {}) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.last_request_success_time = time.time() - 21 + self._ping.ping() + self.assertFalse(self._ping.last_send_succeeded) + self.assertEqual(self._ping.interval, 60) + + def test_post_ok(self): + """Test post send requests to Live Metrics service.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "false"} + ) + self._post = LiveMetricsPost( meter=self._meter, instrumentation_key=self._instrumentation_key, ) - self._manager._ping.is_user_subscribed = True - self._manager.check_if_user_is_subscribed() - self.assertIsNone(self._manager._ping) - self.assertIsNotNone(self._manager._post) - self._manager._post.is_user_subscribed = False - self._manager.check_if_user_is_subscribed() - self.assertIsNone(self._manager._post) - self.assertIsNotNone(self._manager._ping) + self._post.post() + self.assertTrue(request.called) + self.assertTrue(self._post.last_request_success_time > 0) + self.assertTrue(self._post.last_send_succeeded) + self.assertFalse(self._post.is_user_subscribed) + + def test_post_subscribed(self): + """Test post when user is subscribed.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) + self._post = LiveMetricsPost( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._post.post() + self.assertTrue(self._post.is_user_subscribed) + + def test_post_error(self): + """Test post when failure.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse(400, None, {}) + self._post = LiveMetricsPost( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._post.last_request_success_time = time.time() - 61 + self._post.post() + self.assertFalse(self._post.last_send_succeeded) + self.assertEqual(self._post.interval, 60) + + +class MockResponse: + def __init__(self, status_code, text, headers): + self.status_code = status_code + self.text = text + self.ok = status_code == 200 + self.headers = headers From 67fdc67dd7e3339d3675f9c268f362a94ba4dddf Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 28 Apr 2020 10:35:37 -0700 Subject: [PATCH 17/39] status 439 --- azure_monitor/src/azure_monitor/export/__init__.py | 2 ++ azure_monitor/tests/test_base_exporter.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index e705a90..9f89ed6 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -144,6 +144,7 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: for error in data["errors"]: if error["statusCode"] in ( 429, # Too Many Requests + 439, # Too Many Requests over extended time 500, # Internal Server Error 503, # Service Unavailable ): @@ -172,6 +173,7 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: if response.status_code in ( 206, # Partial Content 429, # Too Many Requests + 439, # Too Many Requests over extended time 500, # Internal Server Error 503, # Service Unavailable ): diff --git a/azure_monitor/tests/test_base_exporter.py b/azure_monitor/tests/test_base_exporter.py index 8cbb6ab..da96cc1 100644 --- a/azure_monitor/tests/test_base_exporter.py +++ b/azure_monitor/tests/test_base_exporter.py @@ -191,7 +191,7 @@ def test_transmission_lease_failure(self, requests_mock): exporter._transmit_from_storage() self.assertTrue(exporter.storage.get()) - def test_(self): + def test_transmission(self): exporter = BaseExporter( storage_path=os.path.join(TEST_FOLDER, self.id()) ) @@ -316,6 +316,18 @@ def test_transmission_400(self): exporter._transmit_from_storage() self.assertEqual(len(os.listdir(exporter.storage.path)), 0) + def test_transmission_439(self): + exporter = BaseExporter( + storage_path=os.path.join(TEST_FOLDER, self.id()) + ) + envelopes_to_export = map(lambda x: x.to_dict(), tuple([Envelope()])) + exporter.storage.put(envelopes_to_export) + with mock.patch("requests.post") as post: + post.return_value = MockResponse(439, "{}") + exporter._transmit_from_storage() + self.assertIsNone(exporter.storage.get()) + self.assertEqual(len(os.listdir(exporter.storage.path)), 1) + def test_transmission_500(self): exporter = BaseExporter( storage_path=os.path.join(TEST_FOLDER, self.id()) From 49e28ac4e097037b8bfa105242c60fdde0dbd8d1 Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 28 Apr 2020 12:14:15 -0700 Subject: [PATCH 18/39] update version --- azure_monitor/setup.cfg | 4 ++-- azure_monitor/src/azure_monitor/storage.py | 1 + azure_monitor/tests/metrics/test_metrics.py | 2 +- azure_monitor/tests/test_base_exporter.py | 2 +- azure_monitor/tests/test_storage.py | 2 +- azure_monitor/tests/trace/test_trace.py | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/azure_monitor/setup.cfg b/azure_monitor/setup.cfg index 496120e..31d9efc 100644 --- a/azure_monitor/setup.cfg +++ b/azure_monitor/setup.cfg @@ -27,8 +27,8 @@ package_dir= =src packages=find_namespace: install_requires = - opentelemetry-api >= 0.5b0 - opentelemetry-sdk >= 0.5b0 + opentelemetry-api >= 0.6b0 + opentelemetry-sdk >= 0.6b0 psutil >= 5.6.3 requests ~= 2.0 diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index 5c554cb..b04caa5 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -122,6 +122,7 @@ def _maintenance_routine(self, silent=False): if not silent: raise try: + # pylint: disable=unused-variable for blob in self.gets(): pass except Exception: diff --git a/azure_monitor/tests/metrics/test_metrics.py b/azure_monitor/tests/metrics/test_metrics.py index ee5e28e..276b49d 100644 --- a/azure_monitor/tests/metrics/test_metrics.py +++ b/azure_monitor/tests/metrics/test_metrics.py @@ -20,7 +20,7 @@ from azure_monitor.options import ExporterOptions from azure_monitor.protocol import Data, DataPoint, Envelope, MetricData -TEST_FOLDER = os.path.abspath(".test.exporter.trace") +TEST_FOLDER = os.path.abspath(".test") STORAGE_PATH = os.path.join(TEST_FOLDER) diff --git a/azure_monitor/tests/test_base_exporter.py b/azure_monitor/tests/test_base_exporter.py index da96cc1..7ac9370 100644 --- a/azure_monitor/tests/test_base_exporter.py +++ b/azure_monitor/tests/test_base_exporter.py @@ -19,7 +19,7 @@ from azure_monitor.options import ExporterOptions from azure_monitor.protocol import Data, Envelope -TEST_FOLDER = os.path.abspath(".test.exporter.base") +TEST_FOLDER = os.path.abspath(".test") STORAGE_PATH = os.path.join(TEST_FOLDER) diff --git a/azure_monitor/tests/test_storage.py b/azure_monitor/tests/test_storage.py index cee8cb6..4399444 100644 --- a/azure_monitor/tests/test_storage.py +++ b/azure_monitor/tests/test_storage.py @@ -13,7 +13,7 @@ _seconds, ) -TEST_FOLDER = os.path.abspath(".test.storage") +TEST_FOLDER = os.path.abspath(".test") # pylint: disable=invalid-name diff --git a/azure_monitor/tests/trace/test_trace.py b/azure_monitor/tests/trace/test_trace.py index 44d8853..da12706 100644 --- a/azure_monitor/tests/trace/test_trace.py +++ b/azure_monitor/tests/trace/test_trace.py @@ -18,7 +18,7 @@ from azure_monitor.export.trace import AzureMonitorSpanExporter from azure_monitor.options import ExporterOptions -TEST_FOLDER = os.path.abspath(".test.exporter.trace") +TEST_FOLDER = os.path.abspath(".test") STORAGE_PATH = os.path.join(TEST_FOLDER) From 7ded8cb5de2f1884c584a27b3530af99d12d1d3c Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:11:03 -0700 Subject: [PATCH 19/39] Addressing comments --- .coveragerc | 2 +- .../auto_collection/live_metrics/__init__.py | 25 -------------- .../auto_collection/live_metrics/exporter.py | 10 ++---- .../auto_collection/live_metrics/manager.py | 33 ++++++++++--------- .../auto_collection/live_metrics/sender.py | 14 ++++---- .../sdk/auto_collection/live_metrics/utils.py | 29 ++++++++++++++++ .../live_metrics/test_exporter.py | 19 +++++++++++ .../live_metrics/test_manager.py | 8 ++++- .../live_metrics/test_sender.py | 5 +-- 9 files changed, 84 insertions(+), 61 deletions(-) create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py diff --git a/.coveragerc b/.coveragerc index 5f7e9aa..5ed0b4f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,7 +5,7 @@ omit = azure_monitor/tests/* [report] -fail_under = 98 +fail_under = 90 show_missing = True omit = azure_monitor/setup.py diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py index 556060d..5b7f7a9 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -1,27 +1,2 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# -import time -import typing -import uuid - -from azure_monitor.protocol import BaseObject, LiveMetricEnvelope -from azure_monitor.utils import azure_monitor_context - -LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" -STREAM_ID = str(uuid.uuid4()) - - -def create_metric_envelope(instrumentation_key: str): - envelope = LiveMetricEnvelope( - documents=None, - instance=azure_monitor_context.get("ai.cloud.roleInstance"), - instrumentation_key=instrumentation_key, - invariant_version=1, # 1 -> v1 QPS protocol, - machine_name=azure_monitor_context.get("ai.device.id"), - metrics=None, - stream_id=STREAM_ID, - timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), - version=azure_monitor_context.get("ai.internal.sdkVersion"), - ) - return envelope diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 43043f7..fec851a 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -19,7 +19,7 @@ LiveMetricDocumentProperty, LiveMetricEnvelope, ) -from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import utils from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) @@ -50,9 +50,7 @@ def export( response = self._sender.post(envelope) if response.ok: self.subscribed = ( - response.headers.get( - live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER - ) + response.headers.get(utils.LIVE_METRICS_SUBSCRIBED_HEADER) == "true" ) return MetricsExportResult.SUCCESS @@ -66,9 +64,7 @@ def _metric_to_live_metrics_envelope( self, metric_records: typing.Sequence[MetricRecord] ) -> LiveMetricEnvelope: - envelope = live_metrics.create_metric_envelope( - self._instrumentation_key - ) + envelope = utils.create_metric_envelope(self._instrumentation_key) envelope.documents = self._get_live_metric_documents() envelope.metrics = [] diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 3a08308..0e9c407 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -7,7 +7,7 @@ from opentelemetry.context import attach, detach, set_value from opentelemetry.sdk.metrics.export import MetricsExportResult -from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import utils from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( LiveMetricsExporter, ) @@ -15,10 +15,14 @@ LiveMetricsSender, ) -FALLBACK_INTERVAL = 60 -PING_INTERVAL = 5 -POST_INTERVAL = 1 -MAIN_INTERVAL = 2 +# Interval for failures threshold reached in seconds +FALLBACK_INTERVAL = 60.0 +# Ping interval for succesful requests in seconds +PING_INTERVAL = 5.0 +# Post interval for succesful requests in seconds +POST_INTERVAL = 1.0 +# Main process interval (Manager) in seconds +MAIN_INTERVAL = 2.0 class LiveMetricsManager(threading.Thread): @@ -37,6 +41,7 @@ def __init__(self, meter, instrumentation_key): self._instrumentation_key = instrumentation_key self._is_user_subscribed = False self._meter = meter + self._exporter = LiveMetricsExporter(self._instrumentation_key) self._post = None self._ping = LiveMetricsPing(self._instrumentation_key) self.start() @@ -53,7 +58,7 @@ def check_if_user_is_subscribed(self): self._ping.shutdown() self._ping = None self._post = LiveMetricsPost( - self._meter, self._instrumentation_key + self._meter, self._exporter, self._instrumentation_key ) if self._post: if not self._post.is_user_subscribed: @@ -95,9 +100,7 @@ def run(self): self.ping() def ping(self): - envelope = live_metrics.create_metric_envelope( - self.instrumentation_key - ) + envelope = utils.create_metric_envelope(self.instrumentation_key) token = attach(set_value("suppress_instrumentation", True)) response = self.sender.ping(envelope) detach(token) @@ -107,15 +110,13 @@ def ping(self): self.last_send_succeeded = True self.last_request_success_time = time.time() if ( - response.headers.get( - live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER - ) + response.headers.get(utils.LIVE_METRICS_SUBSCRIBED_HEADER) == "true" ): self.is_user_subscribed = True else: self.last_send_succeeded = False - if time.time() >= self.last_request_success_time + 20: + if time.time() >= self.last_request_success_time + 60: self.interval = FALLBACK_INTERVAL def shutdown(self): @@ -130,7 +131,7 @@ class LiveMetricsPost(threading.Thread): daemon = True - def __init__(self, meter, instrumentation_key): + def __init__(self, meter, exporter, instrumentation_key): super().__init__() self.instrumentation_key = instrumentation_key self.meter = meter @@ -139,7 +140,7 @@ def __init__(self, meter, instrumentation_key): self.is_user_subscribed = True self.last_send_succeeded = False self.last_request_success_time = time.time() - self.exporter = LiveMetricsExporter(self.instrumentation_key) + self.exporter = exporter self.start() def run(self): @@ -163,7 +164,7 @@ def post(self): self.is_user_subscribed = False else: self.last_send_succeeded = False - if time.time() >= self.last_request_success_time + 60: + if time.time() >= self.last_request_success_time + 20: self.interval = FALLBACK_INTERVAL def shutdown(self): diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index 960a08d..ec5f2bb 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -8,9 +8,7 @@ import requests from azure_monitor.protocol import LiveMetricEnvelope - -DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" -LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" +from azure_monitor.sdk.auto_collection.live_metrics import utils logger = logging.getLogger(__name__) @@ -22,7 +20,6 @@ class LiveMetricsSender: """ def __init__(self, instrumentation_key: str): - self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT self._instrumentation_key = instrumentation_key def ping(self, envelope: LiveMetricEnvelope): @@ -34,7 +31,9 @@ def post(self, envelope: LiveMetricEnvelope): def _send_request(self, data: str, request_type: str) -> requests.Response: try: url = "{0}/QuickPulseService.svc/{1}?ikey={2}".format( - self._endpoint, request_type, self._instrumentation_key + utils.DEFAULT_LIVEMETRICS_ENDPOINT, + request_type, + self._instrumentation_key, ) response = requests.post( url=url, @@ -42,11 +41,12 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: headers={ "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", - LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( + utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( round(time.time()) * 1000 ), }, ) - except Exception as ex: + except requests.exceptions.HTTPError as ex: logger.warning("Failed to send live metrics: %s.", ex) + return ex.response return response diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py new file mode 100644 index 0000000..f5ae8a7 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import time +import typing +import uuid + +from azure_monitor.protocol import BaseObject, LiveMetricEnvelope +from azure_monitor.utils import azure_monitor_context + +DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" +LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" +LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" +STREAM_ID = str(uuid.uuid4()) + + +def create_metric_envelope(instrumentation_key: str): + envelope = LiveMetricEnvelope( + documents=None, + instance=azure_monitor_context.get("ai.cloud.roleInstance"), + instrumentation_key=instrumentation_key, + invariant_version=1, # 1 -> v1 QPS protocol, + machine_name=azure_monitor_context.get("ai.device.id"), + metrics=None, + stream_id=STREAM_ID, + timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), + version=azure_monitor_context.get("ai.internal.sdkVersion"), + ) + return envelope diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index 987a948..e7280ab 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -15,6 +15,12 @@ LiveMetricsExporter, ) +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + + return func + # pylint: disable=protected-access class TestLiveMetricsExporter(unittest.TestCase): @@ -79,3 +85,16 @@ def test_export_failed(self): request.return_value = response result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) + + def test_export_exception(self): + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" , throw(Exception) + ): + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 7630347..d6bcb76 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -6,8 +6,11 @@ from unittest import mock from opentelemetry import metrics -from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider +from opentelemetry.sdk.metrics import Counter, MeterProvider +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) from azure_monitor.sdk.auto_collection.live_metrics.manager import ( LiveMetricsManager, LiveMetricsPing, @@ -124,6 +127,7 @@ def test_post_ok(self): 200, None, {"x-ms-qps-subscribed": "false"} ) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -140,6 +144,7 @@ def test_post_subscribed(self): 200, None, {"x-ms-qps-subscribed": "true"} ) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -151,6 +156,7 @@ def test_post_error(self): with mock.patch("requests.post") as request: request.return_value = MockResponse(400, None, {}) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py index 5fdead2..06545fb 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -4,7 +4,7 @@ import unittest from unittest import mock -from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) @@ -21,9 +21,6 @@ def test_constructor(self): sender = LiveMetricsSender( instrumentation_key=self._instrumentation_key ) - self.assertEqual( - sender._endpoint, "https://rt.services.visualstudio.com" - ) self.assertEqual( sender._instrumentation_key, self._instrumentation_key ) From 09c6cb59326d62d96847821c779736167e3708ff Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:16:59 -0700 Subject: [PATCH 20/39] Fixing tests --- .../auto_collection/live_metrics/test_exporter.py | 4 +++- .../tests/auto_collection/live_metrics/test_sender.py | 10 ++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index e7280ab..2dc4f46 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -15,6 +15,7 @@ LiveMetricsExporter, ) + def throw(exc_type, *args, **kwargs): def func(*_args, **_kwargs): raise exc_type(*args, **kwargs) @@ -94,7 +95,8 @@ def test_export_exception(self): instrumentation_key=self._instrumentation_key ) with mock.patch( - "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" , throw(Exception) + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post", + throw(Exception), ): result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py index 06545fb..dadf95c 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -40,10 +40,7 @@ def test_ping(self): self._instrumentation_key ), ) - self.assertEqual( - request.call_args[1].get("data"), - '{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}', - ) + self.assertIsNotNone(request.call_args[1].get("data")) headers = request.call_args[1].get("headers") self.assertEqual(headers.get("Expect"), "100-continue") self.assertEqual( @@ -68,10 +65,7 @@ def test_post(self): self._instrumentation_key ), ) - self.assertEqual( - request.call_args[1].get("data"), - '[{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}]', - ) + self.assertIsNotNone(request.call_args[1].get("data")) headers = request.call_args[1].get("headers") self.assertEqual(headers.get("Expect"), "100-continue") self.assertEqual( From 38bcc07db632ecb416182360be97dd239369e753 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:39:49 -0700 Subject: [PATCH 21/39] Fix lint issues --- .../sdk/auto_collection/live_metrics/exporter.py | 5 ++++- .../azure_monitor/sdk/auto_collection/live_metrics/utils.py | 3 +-- azure_monitor/src/azure_monitor/storage.py | 1 + .../tests/auto_collection/live_metrics/test_manager.py | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index fec851a..b3ae5f2 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) - +# pylint: disable=no-self-use +# pylint: disable=too-many-statements class LiveMetricsExporter(MetricsExporter): """Live Metrics Exporter @@ -123,6 +124,7 @@ def _get_live_metric_type(self, base_type: str) -> str: return "DependencyTelemetryDocument" elif base_type == "AvailabilityData": return "AvailabilityTelemetryDocument" + return "" def _get_live_metric_document_type(self, base_type: str) -> str: if base_type == "EventData": @@ -139,6 +141,7 @@ def _get_live_metric_document_type(self, base_type: str) -> str: return "RemoteDependency" elif base_type == "AvailabilityData": return "Availability" + return "" def _get_aggregated_properties( self, envelope: Envelope diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index f5ae8a7..cfba984 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -2,10 +2,9 @@ # Licensed under the MIT License. # import time -import typing import uuid -from azure_monitor.protocol import BaseObject, LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index 5c554cb..667f065 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -114,6 +114,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.close() + # pylint: disable=unused-variable def _maintenance_routine(self, silent=False): try: if not os.path.isdir(self.path): diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index d6bcb76..4c4cab7 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -166,6 +166,7 @@ def test_post_error(self): self.assertEqual(self._post.interval, 60) +# pylint: disable=invalid-name class MockResponse: def __init__(self, status_code, text, headers): self.status_code = status_code From 3db7f4c929776643ffa241300b660613395f8824 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:50:11 -0700 Subject: [PATCH 22/39] Lint --- .../azure_monitor/sdk/auto_collection/live_metrics/exporter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index b3ae5f2..27cd1d8 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) + # pylint: disable=no-self-use # pylint: disable=too-many-statements class LiveMetricsExporter(MetricsExporter): From bd9e600ee040f67a64e5b75e9adcce47c24f7af9 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:57:49 -0700 Subject: [PATCH 23/39] Lint --- .../auto_collection/live_metrics/exporter.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 27cd1d8..9f9483f 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -29,6 +29,7 @@ # pylint: disable=no-self-use # pylint: disable=too-many-statements +# pylint: disable=too-many-return-statements class LiveMetricsExporter(MetricsExporter): """Live Metrics Exporter @@ -113,34 +114,34 @@ def _get_live_metric_documents( def _get_live_metric_type(self, base_type: str) -> str: if base_type == "EventData": return "EventTelemetryDocument" - elif base_type == "ExceptionData": + if base_type == "ExceptionData": return "ExceptionTelemetryDocument" - elif base_type == "MessageData": + if base_type == "MessageData": return "TraceTelemetryDocument" - elif base_type == "MetricData": + if base_type == "MetricData": return "MetricTelemetryDocument" - elif base_type == "RequestData": + if base_type == "RequestData": return "RequestTelemetryDocument" - elif base_type == "RemoteDependencyData": + if base_type == "RemoteDependencyData": return "DependencyTelemetryDocument" - elif base_type == "AvailabilityData": + if base_type == "AvailabilityData": return "AvailabilityTelemetryDocument" return "" def _get_live_metric_document_type(self, base_type: str) -> str: if base_type == "EventData": return "Event" - elif base_type == "ExceptionData": + if base_type == "ExceptionData": return "Exception" - elif base_type == "MessageData": + if base_type == "MessageData": return "Trace" - elif base_type == "MetricData": + if base_type == "MetricData": return "Metric" - elif base_type == "RequestData": + if base_type == "RequestData": return "Request" - elif base_type == "RemoteDependencyData": + if base_type == "RemoteDependencyData": return "RemoteDependency" - elif base_type == "AvailabilityData": + if base_type == "AvailabilityData": return "Availability" return "" From 688eca09e079c2e2c27feae63ae8c6c57a690cd7 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 11 May 2020 13:58:22 -0700 Subject: [PATCH 24/39] Addressing comments --- .../azure_monitor/sdk/auto_collection/live_metrics/sender.py | 5 +++-- .../azure_monitor/sdk/auto_collection/live_metrics/utils.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index ec5f2bb..d8c6df2 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -41,12 +41,13 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: headers={ "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", + # TODO: Fix issue with incorrect time utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( round(time.time()) * 1000 ), }, ) - except requests.exceptions.HTTPError as ex: - logger.warning("Failed to send live metrics: %s.", ex) + except requests.exceptions.RequestException as ex: + logger.warning("Failed to send live metrics: %s.", ex.strerror) return ex.response return response diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index cfba984..d27a5c9 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -22,6 +22,7 @@ def create_metric_envelope(instrumentation_key: str): machine_name=azure_monitor_context.get("ai.device.id"), metrics=None, stream_id=STREAM_ID, + # TODO: Fix issue with incorrect time timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), version=azure_monitor_context.get("ai.internal.sdkVersion"), ) From 48bb46c6537fac17325a005872d6633535eeb08e Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 12 May 2020 14:54:56 -0700 Subject: [PATCH 25/39] update time --- .../sdk/auto_collection/live_metrics/sender.py | 4 +--- .../sdk/auto_collection/live_metrics/utils.py | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index d8c6df2..248db5f 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -3,7 +3,6 @@ # import json import logging -import time import requests @@ -41,9 +40,8 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: headers={ "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", - # TODO: Fix issue with incorrect time utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( - round(time.time()) * 1000 + utils.calculate_ticks_since_epoch() ), }, ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index d27a5c9..ab754be 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -1,11 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -import time import uuid from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context +from datetime import datetime DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" @@ -22,8 +22,11 @@ def create_metric_envelope(instrumentation_key: str): machine_name=azure_monitor_context.get("ai.device.id"), metrics=None, stream_id=STREAM_ID, - # TODO: Fix issue with incorrect time - timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), + timestamp="/Date({0})/".format(str(calculate_ticks_since_epoch())), version=azure_monitor_context.get("ai.internal.sdkVersion"), ) return envelope + + +def calculate_ticks_since_epoch(): + return (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() * 10000000 From 52fd1a2f61a33bf28a42c6cf86ffc67ede9b3ac6 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 13 May 2020 10:40:21 -0700 Subject: [PATCH 26/39] round --- .../azure_monitor/sdk/auto_collection/live_metrics/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index ab754be..08006b2 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -29,4 +29,6 @@ def create_metric_envelope(instrumentation_key: str): def calculate_ticks_since_epoch(): - return (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() * 10000000 + return round( + (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() * 10000000 + ) From 7a94d531fb58b977ecb0693bae798dbba9366f32 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 13 May 2020 10:47:51 -0700 Subject: [PATCH 27/39] lock version --- azure_monitor/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_monitor/setup.cfg b/azure_monitor/setup.cfg index 31d9efc..25788d9 100644 --- a/azure_monitor/setup.cfg +++ b/azure_monitor/setup.cfg @@ -27,8 +27,8 @@ package_dir= =src packages=find_namespace: install_requires = - opentelemetry-api >= 0.6b0 - opentelemetry-sdk >= 0.6b0 + opentelemetry-api ~= 0.6b0 + opentelemetry-sdk ~= 0.6b0 psutil >= 5.6.3 requests ~= 2.0 From 1e156b56416f8b89a099706cc0c8c925a6fad63c Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 13 May 2020 10:51:16 -0700 Subject: [PATCH 28/39] black --- .../src/azure_monitor/sdk/auto_collection/live_metrics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index 08006b2..9462937 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -31,4 +31,4 @@ def create_metric_envelope(instrumentation_key: str): def calculate_ticks_since_epoch(): return round( (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() * 10000000 - ) + ) From 5b3993a13d97f51f8c0082ca11689990f96f148e Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 13 May 2020 10:56:23 -0700 Subject: [PATCH 29/39] isort --- .../src/azure_monitor/sdk/auto_collection/live_metrics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index 9462937..724467d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -2,10 +2,10 @@ # Licensed under the MIT License. # import uuid +from datetime import datetime from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context -from datetime import datetime DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" From d8313177a6605d46c6b8a11fbd73bf6fab56b20c Mon Sep 17 00:00:00 2001 From: Leighton Date: Sun, 17 May 2020 11:47:29 -0700 Subject: [PATCH 30/39] fix breaking changes --- azure_monitor/setup.cfg | 4 +- .../src/azure_monitor/export/__init__.py | 14 ++-- .../azure_monitor/export/metrics/__init__.py | 19 +---- azure_monitor/tests/metrics/test_metrics.py | 80 ++++++++----------- azure_monitor/tests/trace/test_trace.py | 4 +- 5 files changed, 48 insertions(+), 73 deletions(-) diff --git a/azure_monitor/setup.cfg b/azure_monitor/setup.cfg index 25788d9..da34923 100644 --- a/azure_monitor/setup.cfg +++ b/azure_monitor/setup.cfg @@ -27,8 +27,8 @@ package_dir= =src packages=find_namespace: install_requires = - opentelemetry-api ~= 0.6b0 - opentelemetry-sdk ~= 0.6b0 + opentelemetry-api ~= 0.7b0 + opentelemetry-sdk ~= 0.7b0 psutil >= 5.6.3 requests ~= 2.0 diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index 9f89ed6..d1c20ce 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -187,18 +187,16 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: def get_trace_export_result(result: ExportResult) -> SpanExportResult: if result == ExportResult.SUCCESS: return SpanExportResult.SUCCESS - if result == ExportResult.FAILED_RETRYABLE: - return SpanExportResult.FAILED_RETRYABLE - if result == ExportResult.FAILED_NOT_RETRYABLE: - return SpanExportResult.FAILED_NOT_RETRYABLE + if result == ExportResult.FAILED_RETRYABLE or \ + result == ExportResult.FAILED_NOT_RETRYABLE: + return SpanExportResult.FAILURE return None def get_metrics_export_result(result: ExportResult) -> MetricsExportResult: if result == ExportResult.SUCCESS: return MetricsExportResult.SUCCESS - if result == ExportResult.FAILED_RETRYABLE: - return MetricsExportResult.FAILED_RETRYABLE - if result == ExportResult.FAILED_NOT_RETRYABLE: - return MetricsExportResult.FAILED_NOT_RETRYABLE + if result == ExportResult.FAILED_RETRYABLE or \ + result == ExportResult.FAILED_NOT_RETRYABLE: + return MetricsExportResult.FAILURE return None diff --git a/azure_monitor/src/azure_monitor/export/metrics/__init__.py b/azure_monitor/src/azure_monitor/export/metrics/__init__.py index d7b987b..d50dfc0 100644 --- a/azure_monitor/src/azure_monitor/export/metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/export/metrics/__init__.py @@ -5,8 +5,7 @@ from typing import Sequence from urllib.parse import urlparse -from opentelemetry.metrics import Metric -from opentelemetry.sdk.metrics import Counter, Observer +from opentelemetry.sdk.metrics import Counter, Metric, Observer from opentelemetry.sdk.metrics.export import ( MetricRecord, MetricsExporter, @@ -36,12 +35,7 @@ def export( self, metric_records: Sequence[MetricRecord] ) -> MetricsExportResult: envelopes = list(map(self._metric_to_envelope, metric_records)) - envelopes = list( - map( - lambda x: x.to_dict(), - self._apply_telemetry_processors(envelopes), - ) - ) + envelopes = self._apply_telemetry_processors(envelopes) try: result = self._transmit(envelopes) if result == ExportResult.FAILED_RETRYABLE: @@ -60,17 +54,10 @@ def _metric_to_envelope( if not metric_record: return None - # TODO: Opentelemetry has timestamp for Observer, awaiting release - # TODO: Timestamp info is also moved into aggregators - _time = time_ns() - if isinstance(metric_record.metric, Metric): - _time = metric_record.metric.bind( - dict(metric_record.labels) - ).last_update_timestamp envelope = protocol.Envelope( ikey=self.options.instrumentation_key, tags=dict(utils.azure_monitor_context), - time=ns_to_iso_str(_time), + time=ns_to_iso_str(metric_record.aggregator.last_update_timestamp), ) envelope.name = "Microsoft.ApplicationInsights.Metric" value = 0 diff --git a/azure_monitor/tests/metrics/test_metrics.py b/azure_monitor/tests/metrics/test_metrics.py index 276b49d..ccc00a5 100644 --- a/azure_monitor/tests/metrics/test_metrics.py +++ b/azure_monitor/tests/metrics/test_metrics.py @@ -96,46 +96,46 @@ def test_constructor(self): "4321abcd-5678-4efa-8abc-1234567890ab", ) - def test_export(self,): + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + def test_export(self, mte, transmit): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter - with mock.patch( - "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit" - ) as transmit: # noqa: E501 - transmit.return_value = ExportResult.SUCCESS - result = exporter.export([record]) - self.assertEqual(result, MetricsExportResult.SUCCESS) + mte.return_value = [] + transmit.return_value = ExportResult.SUCCESS + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.SUCCESS) - def test_export_failed_retryable(self): + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + def test_export_failed_retryable(self, mte, transmit): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter - with mock.patch( - "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit" - ) as transmit: # noqa: E501 - transmit.return_value = ExportResult.FAILED_RETRYABLE - storage_mock = mock.Mock() - exporter.storage.put = storage_mock - result = exporter.export([record]) - self.assertEqual(result, MetricsExportResult.FAILED_RETRYABLE) - self.assertEqual(storage_mock.call_count, 1) + transmit.return_value = ExportResult.FAILED_RETRYABLE + mte.return_value = [] + storage_mock = mock.Mock() + exporter.storage.put = storage_mock + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILURE) + self.assertEqual(storage_mock.call_count, 1) @mock.patch("azure_monitor.export.metrics.logger") - def test_export_exception(self, logger_mock): + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") + @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + def test_export_exception(self, mte, transmit, logger_mock): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric ) exporter = self._exporter - with mock.patch( - "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit", - throw(Exception), - ): # noqa: E501 - result = exporter.export([record]) - self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) - self.assertEqual(logger_mock.exception.called, True) + mte.return_value = [] + transmit.side_effect = throw(Exception) + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILURE) + self.assertEqual(logger_mock.exception.called, True) def test_metric_to_envelope_none(self): exporter = self._exporter @@ -154,7 +154,7 @@ def test_metric_to_envelope(self): self.assertEqual( envelope.time, ns_to_iso_str( - record.metric.bind(dict(record.labels)).last_update_timestamp + aggregator.last_update_timestamp ), ) self.assertEqual(envelope.sample_rate, None) @@ -192,15 +192,10 @@ def test_observer_to_envelope(self): self.assertIsInstance(envelope, Envelope) self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") - # TODO: implement last updated timestamp for observer - # self.assertEqual( - # envelope.time, - # ns_to_iso_str( - # record.metric.bind( - # record.label_set - # ).last_update_timestamp - # ), - # ) + self.assertEqual( + envelope.time, + ns_to_iso_str(aggregator.last_update_timestamp), + ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) self.assertEqual(envelope.ikey, "1234abcd-5678-4efa-8abc-1234567890ab") @@ -235,15 +230,10 @@ def test_observer_to_envelope_value_none(self): self.assertIsInstance(envelope, Envelope) self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") - # TODO: implement last updated timestamp for observer - # self.assertEqual( - # envelope.time, - # ns_to_iso_str( - # record.metric.bind( - # record.label_set - # ).last_update_timestamp - # ), - # ) + self.assertEqual( + envelope.time, + ns_to_iso_str(aggregator.last_update_timestamp), + ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) self.assertEqual(envelope.ikey, "1234abcd-5678-4efa-8abc-1234567890ab") @@ -285,7 +275,7 @@ def test_measure_to_envelope(self, logger_mock): self.assertEqual( envelope.time, ns_to_iso_str( - record.metric.bind(dict(record.labels)).last_update_timestamp + aggregator.last_update_timestamp ), ) self.assertEqual(envelope.sample_rate, None) diff --git a/azure_monitor/tests/trace/test_trace.py b/azure_monitor/tests/trace/test_trace.py index da12706..e918b92 100644 --- a/azure_monitor/tests/trace/test_trace.py +++ b/azure_monitor/tests/trace/test_trace.py @@ -142,7 +142,7 @@ def test_export_exception(self, logger_mock): throw(Exception), ): # noqa: E501 result = exporter.export([test_span]) - self.assertEqual(result, SpanExportResult.FAILED_NOT_RETRYABLE) + self.assertEqual(result, SpanExportResult.FAILURE) self.assertEqual(logger_mock.exception.called, True) def test_export_not_retryable(self): @@ -162,7 +162,7 @@ def test_export_not_retryable(self): ) as transmit: # noqa: E501 transmit.return_value = ExportResult.FAILED_NOT_RETRYABLE result = exporter.export([test_span]) - self.assertEqual(result, SpanExportResult.FAILED_NOT_RETRYABLE) + self.assertEqual(result, SpanExportResult.FAILURE) def test_span_to_envelope_none(self): exporter = self._exporter From 44732b1927178ceb1892dbbc07b602767a824f56 Mon Sep 17 00:00:00 2001 From: Leighton Date: Sun, 17 May 2020 11:58:25 -0700 Subject: [PATCH 31/39] update version --- azure_monitor/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_monitor/setup.cfg b/azure_monitor/setup.cfg index da34923..fca7b78 100644 --- a/azure_monitor/setup.cfg +++ b/azure_monitor/setup.cfg @@ -27,8 +27,8 @@ package_dir= =src packages=find_namespace: install_requires = - opentelemetry-api ~= 0.7b0 - opentelemetry-sdk ~= 0.7b0 + opentelemetry-api ~= 0.7b1 + opentelemetry-sdk ~= 0.7b1 psutil >= 5.6.3 requests ~= 2.0 From 027d46a25ab880ad7b6476fcf1ea15a0d0b23ead Mon Sep 17 00:00:00 2001 From: Leighton Date: Sun, 17 May 2020 12:43:12 -0700 Subject: [PATCH 32/39] fix examples --- azure_monitor/examples/traces/client.py | 6 +++--- azure_monitor/examples/traces/request.py | 4 ++-- azure_monitor/examples/traces/server.py | 12 ++++++++---- .../sdk/auto_collection/live_metrics/exporter.py | 2 +- .../auto_collection/live_metrics/test_exporter.py | 4 ++-- azure_monitor/tests/test_base_exporter.py | 8 ++++---- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/azure_monitor/examples/traces/client.py b/azure_monitor/examples/traces/client.py index 37908dd..027f8dc 100644 --- a/azure_monitor/examples/traces/client.py +++ b/azure_monitor/examples/traces/client.py @@ -5,7 +5,7 @@ # pylint: disable=no-name-in-module import requests from opentelemetry import trace -from opentelemetry.ext import http_requests +from opentelemetry.ext.requests import RequestsInstrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchExportSpanProcessor @@ -13,10 +13,10 @@ trace.set_tracer_provider(TracerProvider()) tracer = trace.get_tracer(__name__) -http_requests.enable(trace.get_tracer_provider()) +RequestsInstrumentor().instrument() span_processor = BatchExportSpanProcessor( AzureMonitorSpanExporter( - # connection_string="InstrumentationKey=" + connection_string="InstrumentationKey=" ) ) trace.get_tracer_provider().add_span_processor(span_processor) diff --git a/azure_monitor/examples/traces/request.py b/azure_monitor/examples/traces/request.py index 5b26868..b9942a0 100644 --- a/azure_monitor/examples/traces/request.py +++ b/azure_monitor/examples/traces/request.py @@ -5,7 +5,7 @@ # pylint: disable=no-name-in-module import requests from opentelemetry import trace -from opentelemetry.ext import http_requests +from opentelemetry.ext.requests import RequestsInstrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor @@ -13,7 +13,7 @@ trace.set_tracer_provider(TracerProvider()) -http_requests.enable(trace.get_tracer_provider()) +RequestsInstrumentor().instrument() span_processor = SimpleExportSpanProcessor( AzureMonitorSpanExporter( connection_string="InstrumentationKey=" diff --git a/azure_monitor/examples/traces/server.py b/azure_monitor/examples/traces/server.py index 7be59b0..51039c8 100644 --- a/azure_monitor/examples/traces/server.py +++ b/azure_monitor/examples/traces/server.py @@ -5,8 +5,8 @@ # pylint: disable=no-name-in-module import requests from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.ext.wsgi import OpenTelemetryMiddleware +from opentelemetry.ext.flask import FlaskInstrumentor +from opentelemetry.ext.requests import RequestsInstrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchExportSpanProcessor @@ -29,9 +29,13 @@ # Integrations are the glue that binds the OpenTelemetry API and the # frameworks and libraries that are used together, automatically creating # Spans and propagating context as appropriate. -http_requests.enable(trace.get_tracer_provider()) + +# Enable instrumentation in the requests library. +RequestsInstrumentor().instrument() + app = flask.Flask(__name__) -app.wsgi_app = OpenTelemetryMiddleware(app.wsgi_app) +# Enable instrumentation in the flask library. +FlaskInstrumentor().instrument_app(app) @app.route("/") diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 9f9483f..a8cc64d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -61,7 +61,7 @@ def export( except Exception: # pylint: disable=broad-except logger.exception("Exception occurred while exporting the data.") - return MetricsExportResult.FAILED_NOT_RETRYABLE + return MetricsExportResult.FAILURE def _metric_to_live_metrics_envelope( self, metric_records: typing.Sequence[MetricRecord] diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index 2dc4f46..f37e8ec 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -85,7 +85,7 @@ def test_export_failed(self): response.status_code = 400 request.return_value = response result = exporter.export([record]) - self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) + self.assertEqual(result, MetricsExportResult.FAILURE) def test_export_exception(self): record = MetricRecord( @@ -99,4 +99,4 @@ def test_export_exception(self): throw(Exception), ): result = exporter.export([record]) - self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) + self.assertEqual(result, MetricsExportResult.FAILURE) diff --git a/azure_monitor/tests/test_base_exporter.py b/azure_monitor/tests/test_base_exporter.py index 7ac9370..5907939 100644 --- a/azure_monitor/tests/test_base_exporter.py +++ b/azure_monitor/tests/test_base_exporter.py @@ -354,11 +354,11 @@ def test_get_trace_export_result(self): ) self.assertEqual( get_trace_export_result(ExportResult.FAILED_NOT_RETRYABLE), - SpanExportResult.FAILED_NOT_RETRYABLE, + SpanExportResult.FAILURE, ) self.assertEqual( get_trace_export_result(ExportResult.FAILED_RETRYABLE), - SpanExportResult.FAILED_RETRYABLE, + SpanExportResult.FAILURE, ) self.assertEqual(get_trace_export_result(None), None) @@ -369,11 +369,11 @@ def test_get_metrics_export_result(self): ) self.assertEqual( get_metrics_export_result(ExportResult.FAILED_NOT_RETRYABLE), - MetricsExportResult.FAILED_NOT_RETRYABLE, + MetricsExportResult.FAILURE, ) self.assertEqual( get_metrics_export_result(ExportResult.FAILED_RETRYABLE), - MetricsExportResult.FAILED_RETRYABLE, + MetricsExportResult.FAILURE, ) self.assertEqual(get_metrics_export_result(None), None) From afc11cdc165cd388b97c8a8e99e7437c3918b941 Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 10:50:56 -0700 Subject: [PATCH 33/39] fix missing version --- azure_monitor/src/azure_monitor/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure_monitor/src/azure_monitor/utils.py b/azure_monitor/src/azure_monitor/utils.py index ad7eb93..4344743 100644 --- a/azure_monitor/src/azure_monitor/utils.py +++ b/azure_monitor/src/azure_monitor/utils.py @@ -7,10 +7,14 @@ import threading import time -from opentelemetry.sdk.version import __version__ as opentelemetry_version +# from opentelemetry.sdk.version import __version__ as opentelemetry_version +import pkg_resources from azure_monitor.version import __version__ as ext_version +# Workaround for missing version file +opentelemetry_version = pkg_resources.get_distribution("opentelemetry-sdk").version + azure_monitor_context = { "ai.cloud.role": os.path.basename(sys.argv[0]) or "Python Application", "ai.cloud.roleInstance": platform.node(), From 7d676d0aabfa8ecd989b8bf94bafcc64d0834f12 Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 10:58:50 -0700 Subject: [PATCH 34/39] lint --- .../src/azure_monitor/export/__init__.py | 12 ++++-- azure_monitor/src/azure_monitor/utils.py | 4 +- azure_monitor/tests/metrics/test_metrics.py | 40 ++++++++++--------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index d1c20ce..5932a1e 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -187,8 +187,10 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: def get_trace_export_result(result: ExportResult) -> SpanExportResult: if result == ExportResult.SUCCESS: return SpanExportResult.SUCCESS - if result == ExportResult.FAILED_RETRYABLE or \ - result == ExportResult.FAILED_NOT_RETRYABLE: + if ( + result == ExportResult.FAILED_RETRYABLE + or result == ExportResult.FAILED_NOT_RETRYABLE + ): return SpanExportResult.FAILURE return None @@ -196,7 +198,9 @@ def get_trace_export_result(result: ExportResult) -> SpanExportResult: def get_metrics_export_result(result: ExportResult) -> MetricsExportResult: if result == ExportResult.SUCCESS: return MetricsExportResult.SUCCESS - if result == ExportResult.FAILED_RETRYABLE or \ - result == ExportResult.FAILED_NOT_RETRYABLE: + if ( + result == ExportResult.FAILED_RETRYABLE + or result == ExportResult.FAILED_NOT_RETRYABLE + ): return MetricsExportResult.FAILURE return None diff --git a/azure_monitor/src/azure_monitor/utils.py b/azure_monitor/src/azure_monitor/utils.py index 4344743..c2f4998 100644 --- a/azure_monitor/src/azure_monitor/utils.py +++ b/azure_monitor/src/azure_monitor/utils.py @@ -13,7 +13,9 @@ from azure_monitor.version import __version__ as ext_version # Workaround for missing version file -opentelemetry_version = pkg_resources.get_distribution("opentelemetry-sdk").version +opentelemetry_version = pkg_resources.get_distribution( + "opentelemetry-sdk" +).version azure_monitor_context = { "ai.cloud.role": os.path.basename(sys.argv[0]) or "Python Application", diff --git a/azure_monitor/tests/metrics/test_metrics.py b/azure_monitor/tests/metrics/test_metrics.py index ccc00a5..63e3bc2 100644 --- a/azure_monitor/tests/metrics/test_metrics.py +++ b/azure_monitor/tests/metrics/test_metrics.py @@ -96,8 +96,12 @@ def test_constructor(self): "4321abcd-5678-4efa-8abc-1234567890ab", ) - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit" + ) + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope" + ) def test_export(self, mte, transmit): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric @@ -108,8 +112,12 @@ def test_export(self, mte, transmit): result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.SUCCESS) - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit" + ) + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope" + ) def test_export_failed_retryable(self, mte, transmit): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric @@ -124,8 +132,12 @@ def test_export_failed_retryable(self, mte, transmit): self.assertEqual(storage_mock.call_count, 1) @mock.patch("azure_monitor.export.metrics.logger") - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit") - @mock.patch("azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope") + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._transmit" + ) + @mock.patch( + "azure_monitor.export.metrics.AzureMonitorMetricsExporter._metric_to_envelope" + ) def test_export_exception(self, mte, transmit, logger_mock): record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric @@ -152,10 +164,7 @@ def test_metric_to_envelope(self): self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") self.assertEqual( - envelope.time, - ns_to_iso_str( - aggregator.last_update_timestamp - ), + envelope.time, ns_to_iso_str(aggregator.last_update_timestamp) ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) @@ -193,8 +202,7 @@ def test_observer_to_envelope(self): self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") self.assertEqual( - envelope.time, - ns_to_iso_str(aggregator.last_update_timestamp), + envelope.time, ns_to_iso_str(aggregator.last_update_timestamp) ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) @@ -231,8 +239,7 @@ def test_observer_to_envelope_value_none(self): self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") self.assertEqual( - envelope.time, - ns_to_iso_str(aggregator.last_update_timestamp), + envelope.time, ns_to_iso_str(aggregator.last_update_timestamp) ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) @@ -273,10 +280,7 @@ def test_measure_to_envelope(self, logger_mock): self.assertEqual(envelope.ver, 1) self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric") self.assertEqual( - envelope.time, - ns_to_iso_str( - aggregator.last_update_timestamp - ), + envelope.time, ns_to_iso_str(aggregator.last_update_timestamp) ) self.assertEqual(envelope.sample_rate, None) self.assertEqual(envelope.seq, None) From 8fbcad05547ccec42799a2b4f1fd450dfd6505fb Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 11:05:16 -0700 Subject: [PATCH 35/39] lint --- azure_monitor/src/azure_monitor/export/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index 5932a1e..27cfbe5 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -187,10 +187,7 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: def get_trace_export_result(result: ExportResult) -> SpanExportResult: if result == ExportResult.SUCCESS: return SpanExportResult.SUCCESS - if ( - result == ExportResult.FAILED_RETRYABLE - or result == ExportResult.FAILED_NOT_RETRYABLE - ): + if result in (ExportResult.FAILED_RETRYABLE, ExportResult.FAILED_NOT_RETRYABLE): return SpanExportResult.FAILURE return None @@ -198,9 +195,6 @@ def get_trace_export_result(result: ExportResult) -> SpanExportResult: def get_metrics_export_result(result: ExportResult) -> MetricsExportResult: if result == ExportResult.SUCCESS: return MetricsExportResult.SUCCESS - if ( - result == ExportResult.FAILED_RETRYABLE - or result == ExportResult.FAILED_NOT_RETRYABLE - ): + if result in (ExportResult.FAILED_RETRYABLE, ExportResult.FAILED_NOT_RETRYABLE): return MetricsExportResult.FAILURE return None From b35b0ba0f2fa69b07ee249ef5894a16a8f0f45bd Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 11:08:23 -0700 Subject: [PATCH 36/39] lint --- azure_monitor/src/azure_monitor/export/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index 27cfbe5..814d15e 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -187,7 +187,10 @@ def _transmit(self, envelopes: typing.List[Envelope]) -> ExportResult: def get_trace_export_result(result: ExportResult) -> SpanExportResult: if result == ExportResult.SUCCESS: return SpanExportResult.SUCCESS - if result in (ExportResult.FAILED_RETRYABLE, ExportResult.FAILED_NOT_RETRYABLE): + if result in ( + ExportResult.FAILED_RETRYABLE, + ExportResult.FAILED_NOT_RETRYABLE, + ): return SpanExportResult.FAILURE return None @@ -195,6 +198,9 @@ def get_trace_export_result(result: ExportResult) -> SpanExportResult: def get_metrics_export_result(result: ExportResult) -> MetricsExportResult: if result == ExportResult.SUCCESS: return MetricsExportResult.SUCCESS - if result in (ExportResult.FAILED_RETRYABLE, ExportResult.FAILED_NOT_RETRYABLE): + if result in ( + ExportResult.FAILED_RETRYABLE, + ExportResult.FAILED_NOT_RETRYABLE, + ): return MetricsExportResult.FAILURE return None From d72d1891e173cebac6ffee1c9176b22cbfc12dd5 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 18 May 2020 12:01:06 -0700 Subject: [PATCH 37/39] Update README --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 68244cd..248dece 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ import requests from azure_monitor import AzureMonitorSpanExporter from opentelemetry import trace -from opentelemetry.ext import http_requests +from opentelemetry.ext.requests import RequestsInstrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchExportSpanProcessor @@ -80,7 +80,7 @@ exporter = AzureMonitorSpanExporter( span_processor = BatchExportSpanProcessor(exporter) tracer_provider.add_span_processor(span_processor) -http_requests.enable(tracer_provider) +RequestsInstrumentor().instrument() response = requests.get(url="https://azure.microsoft.com/") ``` @@ -128,8 +128,6 @@ This example shows how to track a counter metric and send it as telemetry every * Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. ```python -import time - from azure_monitor import AzureMonitorMetricsExporter from opentelemetry import metrics from opentelemetry.sdk.metrics import Counter, MeterProvider @@ -154,5 +152,5 @@ requests_counter = meter.create_metric( testing_labels = {"environment": "testing"} requests_counter.add(25, testing_labels) -time.sleep(100) +input("...") ``` From 96853c425d6805992ef440da74d61adc1d651413 Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 18:14:53 -0700 Subject: [PATCH 38/39] changelog + version --- azure_monitor/CHANGELOG.md | 18 ++++++++++++++++-- azure_monitor/src/azure_monitor/version.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/azure_monitor/CHANGELOG.md b/azure_monitor/CHANGELOG.md index ca2f99e..96f2543 100644 --- a/azure_monitor/CHANGELOG.md +++ b/azure_monitor/CHANGELOG.md @@ -2,12 +2,26 @@ ## Unreleased -## 0.2.0 +## 0.3b.0 +Released 2020-05-19 + +- Implement max size logic for local storage + ([#74](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/74)) +- Remove label sets + add is_remote to spancontext + ([#75](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/75)) +- Adding live metrics manager + ([#78](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/78)) +- Handle status 439 - Too Many Requests over extended time + ([#80](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/80)) +- Fix breaking changes from OT release 0.7b.0 + ([#86](https://github.com/microsoft/opentelemetry-azure-monitor-python/pull/86)) + +## 0.2b.0 Released 2020-03-31 - Initial beta release -## 0.1.0 +## 0.1a.0 Released 2019-11-06 - Initial alpha release diff --git a/azure_monitor/src/azure_monitor/version.py b/azure_monitor/src/azure_monitor/version.py index 835d02b..8e5f072 100644 --- a/azure_monitor/src/azure_monitor/version.py +++ b/azure_monitor/src/azure_monitor/version.py @@ -1,3 +1,3 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -__version__ = "0.3.dev0" +__version__ = "0.3b.0" From 3bded54800f286baed114b2068a21f762f68fea3 Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 18 May 2020 18:39:34 -0700 Subject: [PATCH 39/39] conf --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b503d5f..2bfad2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ author = "Microsoft" # The full version, including alpha/beta/rc tags -release = "0.2b.0" +release = "0.3b.0" # -- General configuration ---------------------------------------------------