From 5e6c72419996ca2e598514a96ff8195bd9b6c129 Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 16 Jun 2020 23:55:45 -0700 Subject: [PATCH 1/7] Rename long named tests + refactor tox/nox --- .circleci/config.yml | 2 +- .../tests/test_azure_trace_exporter.py | 6 +- .../examples/app/selery.py | 79 ++++++++++++++++ .../examples/app/tasks.py | 6 ++ noxfile.py | 28 +++++- tox.ini | 90 ++++++++++--------- 6 files changed, 162 insertions(+), 49 deletions(-) create mode 100644 contrib/opencensus-ext-django/examples/app/selery.py create mode 100644 contrib/opencensus-ext-django/examples/app/tasks.py diff --git a/.circleci/config.yml b/.circleci/config.yml index c490c04ef..a23cc35bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: ignore: - gh-pages docker: - - image: googleapis/nox:0.17.0 + - image: googleapis/nox:0.18.2 - image: mysql:5.7 environment: MYSQL_ROOT_HOST: "%" diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index 3543157b5..716c9dc55 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -745,7 +745,7 @@ def test_transmission_nothing(self): exporter._transmit_from_storage() exporter._stop() - def test_transmission_request_exception(self): + def test_transmission_pre_exception(self): exporter = trace_exporter.AzureExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), @@ -771,7 +771,7 @@ def test_transmission_lease_failure(self, requests_mock): self.assertTrue(exporter.storage.get()) exporter._stop() - def test_transmission_response_exception(self): + def test_transmission_exception(self): exporter = trace_exporter.AzureExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), @@ -840,7 +840,7 @@ def test_transmission_206_500(self): self.assertEqual(exporter.storage.get().get(), (3,)) exporter._stop() - def test_transmission_206_nothing_to_retry(self): + def test_transmission_206_no_retry(self): exporter = trace_exporter.AzureExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), diff --git a/contrib/opencensus-ext-django/examples/app/selery.py b/contrib/opencensus-ext-django/examples/app/selery.py new file mode 100644 index 000000000..be8bff336 --- /dev/null +++ b/contrib/opencensus-ext-django/examples/app/selery.py @@ -0,0 +1,79 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging + +from celery import Celery +from celery.signals import setup_logging +from opencensus.ext.azure.log_exporter import AzureLogHandler + +from django.conf import settings + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings') + +app = Celery('app') +app.config_from_object('django.conf:settings', namespace='CELERY') +# app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.autodiscover_tasks() + +# def add_azure_log_handler_to_logger(logger, propagate=True): +# """ +# Given a logger, add a AzureLogHandler to it +# :param logger: +# :param propagate: +# :return: +# """ +# formatter = logging.Formatter("[Celery/%(processName)s] %(message)s") +# # Azure Handler: +# azure_log_handler = AzureLogHandler() +# azure_log_handler.setFormatter(formatter) +# azure_log_handler.setLevel(logging.INFO) +# logger.addHandler(azure_log_handler) +# logger.setLevel(logging.INFO) +# logger.propagate = propagate + +# @setup_logging.connect +# def setup_loggers(*args, **kwargs): +# """ +# Using the celery "setup_logging" signal to override and fully define the logging configuration for Celery +# :param args: +# :param kwargs: +# :return: +# """ +# # Configure Celery logging from the Django settings' logging configuration +# from logging.config import dictConfig +# from django.conf import settings +# dictConfig(settings.LOGGING) + +# # Test the root logger (configured in django settings to log to Azure as well +# logger = logging.getLogger('root') +# logger.warning('TRYING LOGGING FROM [%s]' % logger.name) + +# # Configure the Celery top level logger +# logger = logging.getLogger('celery') +# # Add a local file log handler to make sure we capture every message locally +# logger.addHandler(logging.FileHandler("/data/log/worker/importer.log")) +# # In addition, also manually add a AzureLogHandler to it (duplicate with the root's handler) +# logger = add_azure_log_handler_to_logger(logger, propagate=False) +# # Log a test warning message +# logger.warning('TRYING LOGGING FROM [%s]' % logger.name) + +# # Log a test warning message from a lower-level celery logger +# logger = logging.getLogger('celery.task') +# logger.warning('TRYING LOGGING FROM [%s]' % logger.name) + +# # Log a test warning message from a specific django app task logger +# logger = logging.getLogger('etl.tasks') +# logger.warning('TRYING LOGGING FROM [%s]' % logger.name) diff --git a/contrib/opencensus-ext-django/examples/app/tasks.py b/contrib/opencensus-ext-django/examples/app/tasks.py new file mode 100644 index 000000000..68eac5865 --- /dev/null +++ b/contrib/opencensus-ext-django/examples/app/tasks.py @@ -0,0 +1,6 @@ +from celery import shared_task +import random + +@shared_task +def create(total): + return random.choices([1,2,3]) diff --git a/noxfile.py b/noxfile.py index 113ed7dfd..abf5aa8c4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,9 +14,10 @@ from __future__ import absolute_import -import nox import os +import nox + def _install_dev_packages(session): session.install('-e', 'context/opencensus-context') @@ -49,13 +50,16 @@ def _install_dev_packages(session): def _install_test_dependencies(session): - session.install('mock') + session.install('mock==3.0.5') session.install('pytest==4.6.4') + # 842 - Unit tests failing on CI due to failed import for coverage + # Might have something to do with the CircleCI image + # session.install('pytest-cov') session.install('retrying') session.install('unittest2') -@nox.session(python=['2.7', '3.4', '3.5', '3.6']) +@nox.session(python=['2.7', '3.5', '3.6']) def unit(session): """Run the unit test suite.""" @@ -69,6 +73,13 @@ def unit(session): session.run( 'py.test', '--quiet', + # '--cov=opencensus', + # '--cov=context', + # '--cov=contrib', + # '--cov-append', + # '--cov-config=.coveragerc', + # '--cov-report=', + # '--cov-fail-under=97', 'tests/unit/', 'context/', 'contrib/', @@ -128,6 +139,17 @@ def lint_setup_py(session): 'python', 'setup.py', 'check', '--restructuredtext', '--strict') +# @nox.session(python='3.6') +# def cover(session): +# """Run the final coverage report. +# This outputs the coverage report aggregating coverage from the unit +# test runs (not system test runs), and then erases coverage data. +# """ +# session.install('coverage', 'pytest-cov') +# session.run('coverage', 'report', '--show-missing', '--fail-under=100') +# session.run('coverage', 'erase') + + @nox.session(python='3.6') def docs(session): """Build the docs.""" diff --git a/tox.ini b/tox.ini index 74a05863f..2d16b27b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,53 +1,59 @@ [tox] -envlist = py{27,34,35,36,37}-unit, py37-lint, py37-setup, py37-docs +envlist = + py{27,34,35,36,37}-unit + py37-lint + py37-setup + py37-docs [testenv] install_command = python -m pip install {opts} {packages} deps = - py{27,34,35,36,37}-unit,py37-lint: mock - py{27,34,35,36,37}-unit,py37-lint: pytest==4.6.4 - py{27,34,35,36,37}-unit,py37-lint: pytest-cov - py{27,34,35,36,37}-unit,py37-lint: retrying - py{27,34,35,36,37}-unit,py37-lint: unittest2 - py{27,34,35,36,37}-unit,py37-lint,py37-setup,py37-docs: -e context/opencensus-context - py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e contrib/opencensus-correlation - py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e . - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-azure - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-datadog - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-dbapi - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-django - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-flask - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-gevent - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-grpc - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-httplib - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-jaeger - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-logging - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-mysql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-ocagent - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-postgresql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-prometheus - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pymongo - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pymysql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pyramid - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-requests - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-sqlalchemy - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-stackdriver - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-threading - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-zipkin - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-google-cloud-clientlibs - py37-lint: flake8 - py37-setup: docutils - py37-setup: pygments - py37-docs: setuptools >= 36.4.0 - py37-docs: sphinx >= 1.6.3 + unit,lint: mock==3.0.5 + unit,lint: pytest==4.6.4 + unit,lint: pytest-cov + unit,lint: retrying + unit,lint: unittest2 + unit,lint,py37-setup,docs: -e context/opencensus-context + unit,lint,docs: -e contrib/opencensus-correlation + unit,lint,docs: -e . + unit,lint: -e contrib/opencensus-ext-azure + unit,lint: -e contrib/opencensus-ext-datadog + unit,lint: -e contrib/opencensus-ext-dbapi + unit,lint: -e contrib/opencensus-ext-django + unit,lint: -e contrib/opencensus-ext-flask + unit,lint: -e contrib/opencensus-ext-gevent + unit,lint: -e contrib/opencensus-ext-grpc + unit,lint: -e contrib/opencensus-ext-httplib + unit,lint: -e contrib/opencensus-ext-jaeger + unit,lint: -e contrib/opencensus-ext-logging + unit,lint: -e contrib/opencensus-ext-mysql + unit,lint: -e contrib/opencensus-ext-ocagent + unit,lint: -e contrib/opencensus-ext-postgresql + unit,lint: -e contrib/opencensus-ext-prometheus + unit,lint: -e contrib/opencensus-ext-pymongo + unit,lint: -e contrib/opencensus-ext-pymysql + unit,lint: -e contrib/opencensus-ext-pyramid + unit,lint: -e contrib/opencensus-ext-requests + unit,lint: -e contrib/opencensus-ext-sqlalchemy + unit,lint: -e contrib/opencensus-ext-stackdriver + unit,lint: -e contrib/opencensus-ext-threading + unit,lint: -e contrib/opencensus-ext-zipkin + unit,lint: -e contrib/opencensus-ext-google-cloud-clientlibs + lint: flake8 + lint: isort ~= 4.3.21 + setup: docutils + setup: pygments + docs: setuptools >= 36.4.0 + docs: sphinx >= 1.6.3 commands = - py{27,34,35,36,37}-unit: py.test --quiet --cov={envdir}/opencensus --cov=context --cov=contrib --cov-report term-missing --cov-config=.coveragerc --cov-fail-under=97 tests/unit/ context/ contrib/ - ; TODO: System tests - py37-lint: flake8 context/ contrib/ opencensus/ tests/ examples/ - py37-lint: - bash ./scripts/pylint.sh + unit: py.test --quiet --cov={envdir}/opencensus --cov=context --cov=contrib --cov-report term-missing --cov-config=.coveragerc --cov-fail-under=97 tests/unit/ context/ contrib/ + ; TODO system tests + lint: isort --check-only --diff --recursive . + lint: flake8 context/ contrib/ opencensus/ tests/ examples/ + lint: - bash ./scripts/pylint.sh py37-setup: python setup.py check --restructuredtext --strict py37-docs: bash ./scripts/update_docs.sh ; TODO deployment - + \ No newline at end of file From a09fa0c258eedcd50a9adfd196ebf3545c771a60 Mon Sep 17 00:00:00 2001 From: Leighton Date: Tue, 16 Jun 2020 23:57:15 -0700 Subject: [PATCH 2/7] Implement max size logic for local storage for Azure exporters --- .../opencensus/ext/azure/common/__init__.py | 2 +- .../opencensus/ext/azure/common/storage.py | 34 ++++++++++- .../tests/test_storage.py | 57 +++++++++++++++++-- 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index 6b763c037..ce6cd801b 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -97,7 +97,7 @@ def __init__(self, *args, **kwargs): minimum_retry_interval=60, # minimum retry interval in seconds proxy=None, storage_maintenance_period=60, - storage_max_size=100*1024*1024, + storage_max_size=50*1024*1024, # 50MiB storage_path=os.path.join( os.path.expanduser('~'), '.opencensus', diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py index a7fce53d7..cf9283cc0 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py @@ -1,10 +1,13 @@ import datetime import json -import random +import logging import os +import random from opencensus.common.schedule import PeriodicTask +logger = logging.getLogger(__name__) + def _fmt(timestamp): return timestamp.strftime('%Y-%m-%dT%H%M%S.%f') @@ -77,7 +80,7 @@ class LocalFileStorage(object): def __init__( self, path, - max_size=100*1024*1024, # 100MB + max_size=50*1024*1024, # 50MiB maintenance_period=60, # 1 minute retention_period=7*24*60*60, # 7 days write_timeout=60, # 1 minute @@ -162,6 +165,8 @@ def get(self): return None def put(self, data, lease_period=0, silent=False): + if not self._check_storage_size(): + return None blob = LocalFileBlob(os.path.join( self.path, '{}-{}.blob'.format( @@ -170,3 +175,28 @@ def put(self, data, lease_period=0, silent=False): ), )) return blob.put(data, lease_period=lease_period, silent=silent) + + def _check_storage_size(self): + size = 0 + for dirpath, dirnames, filenames in os.walk(self.path): + for f in filenames: + fp = os.path.join(dirpath, f) + # skip if it is symbolic link + if not os.path.islink(fp): + try: + size += os.path.getsize(fp) + except OSError: + logger.error( + "Path %s does not exist or is inaccessible.", fp + ) + continue + if size >= self.max_size: + logger.warning( + "Persistent storage max capacity has been " + "reached. Currently at %fKB. Telemetry will be " + "lost. Please consider increasing the value of " + "'storage_max_size' in exporter config.", + format(size/1024) + ) + return False + return True diff --git a/contrib/opencensus-ext-azure/tests/test_storage.py b/contrib/opencensus-ext-azure/tests/test_storage.py index 9b9b2e12b..98e1196b3 100644 --- a/contrib/opencensus-ext-azure/tests/test_storage.py +++ b/contrib/opencensus-ext-azure/tests/test_storage.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import os import shutil import unittest -from opencensus.ext.azure.common.storage import _now -from opencensus.ext.azure.common.storage import _seconds -from opencensus.ext.azure.common.storage import LocalFileBlob -from opencensus.ext.azure.common.storage import LocalFileStorage +import mock + +from opencensus.ext.azure.common.storage import ( + LocalFileBlob, + LocalFileStorage, + _now, + _seconds, +) -TEST_FOLDER = os.path.abspath('.test') +TEST_FOLDER = os.path.abspath('.test.storage') def setUpModule(): @@ -113,6 +116,48 @@ def test_put(self): self.assertIsNone(stor.put(input, silent=True)) self.assertRaises(Exception, lambda: stor.put(input)) + def test_put_max_size(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd')) as stor: + size_mock = mock.Mock() + size_mock.return_value = False + stor._check_storage_size = size_mock + stor.put(input) + self.assertEqual(stor.get(), None) + + def test_check_storage_size_full(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd2'), 1) as stor: + stor.put(input) + self.assertFalse(stor._check_storage_size()) + + def test_check_storage_size_not_full(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + stor.put(input) + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_no_files(self): + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_links(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd4'), 1000) as stor: + stor.put(input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_error(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor: + with mock.patch('os.path.getsize', side_effect=throw(OSError)): + stor.put(input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + def test_maintanence_routine(self): with mock.patch('os.makedirs') as m: m.return_value = None From a7fdd6a6013bf692385635a3d2c07b0a9b6d0ba4 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 17 Jun 2020 00:04:47 -0700 Subject: [PATCH 3/7] Azure Transport + storage changes --- README.rst | 4 + contrib/opencensus-ext-azure/CHANGELOG.md | 11 +- .../opencensus/ext/azure/common/__init__.py | 19 +- .../opencensus/ext/azure/common/storage.py | 32 ++- .../opencensus/ext/azure/common/transport.py | 14 +- .../metrics_exporter/standard_metrics/cpu.py | 1 + .../standard_metrics/memory.py | 1 + .../standard_metrics/process.py | 1 + .../tests/test_azure_trace_exporter.py | 181 -------------- ...t_processor.py => test_processor_mixin.py} | 0 .../tests/test_storage.py | 69 +++--- .../tests/test_transport_mixin.py | 225 ++++++++++++++++++ tox.ini | 2 +- 13 files changed, 300 insertions(+), 260 deletions(-) rename contrib/opencensus-ext-azure/tests/{test_processor.py => test_processor_mixin.py} (100%) create mode 100644 contrib/opencensus-ext-azure/tests/test_transport_mixin.py diff --git a/README.rst b/README.rst index e056d4c4d..3532ec6c1 100644 --- a/README.rst +++ b/README.rst @@ -2,11 +2,15 @@ OpenCensus - A stats collection and distributed tracing framework ================================================================= |gitter| +|travisci| |circleci| |pypi| |compat_check_pypi| |compat_check_github| + +.. |travisci| image:: https://travis-ci.org/census-instrumentation/opencensus-python.svg?branch=master + :target: https://travis-ci.org/census-instrumentation/opencensus-python .. |circleci| image:: https://circleci.com/gh/census-instrumentation/opencensus-python.svg?style=shield :target: https://circleci.com/gh/census-instrumentation/opencensus-python .. |gitter| image:: https://badges.gitter.im/census-instrumentation/lobby.svg diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 72cb95326..fcee072cb 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,15 +2,8 @@ ## Unreleased -## 1.0.2 -Released 2020-02-03 - -- Add local storage and retry logic for Azure Metrics Exporter - ([#845](https://github.com/census-instrumentation/opencensus-python/pull/845)) -- Add Fixed-rate sampling logic for Azure Log Exporter - ([#848](https://github.com/census-instrumentation/opencensus-python/pull/848)) -- Implement TelemetryProcessors for Azure exporters - ([#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) +- Change default path of local storage + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) ## 1.0.1 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index ce6cd801b..fcb25acb5 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -13,15 +13,17 @@ # limitations under the License. import os -import sys +import tempfile from opencensus.ext.azure.common.protocol import BaseObject INGESTION_ENDPOINT = 'ingestionendpoint' INSTRUMENTATION_KEY = 'instrumentationkey' +TEMPDIR_PREFIX = "opencensus-python-" def process_options(options): + # Connection string/ikey code_cs = parse_connection_string(options.connection_string) code_ikey = options.instrumentation_key env_cs = parse_connection_string( @@ -46,6 +48,14 @@ def process_options(options): or 'https://dc.services.visualstudio.com' options.endpoint = endpoint + '/v2/track' + # storage path + if options.storage_path is None: + TEMPDIR_SUFFIX = options.instrumentation_key or "" + options.storage_path = os.path.join( + tempfile.gettempdir(), + TEMPDIR_PREFIX + TEMPDIR_SUFFIX + ) + def parse_connection_string(connection_string): if connection_string is None: @@ -98,12 +108,7 @@ def __init__(self, *args, **kwargs): proxy=None, storage_maintenance_period=60, storage_max_size=50*1024*1024, # 50MiB - storage_path=os.path.join( - os.path.expanduser('~'), - '.opencensus', - '.azure', - os.path.basename(sys.argv[0]) or '.console', - ), + storage_path=None, storage_retention_period=7*24*60*60, timeout=10.0, # networking timeout in seconds ) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py index cf9283cc0..304b1f5e3 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py @@ -25,14 +25,13 @@ class LocalFileBlob(object): def __init__(self, fullpath): self.fullpath = fullpath - def delete(self, silent=False): + def delete(self): try: os.remove(self.fullpath) except Exception: - if not silent: - raise + pass # keep silent - def get(self, silent=False): + def get(self): try: with open(self.fullpath, 'r') as file: return tuple( @@ -40,10 +39,9 @@ def get(self, silent=False): for line in file.readlines() ) except Exception: - if not silent: - raise + pass # keep silent - def put(self, data, lease_period=0, silent=False): + def put(self, data, lease_period=0): try: fullpath = self.fullpath + '.tmp' with open(fullpath, 'w') as file: @@ -59,8 +57,7 @@ def put(self, data, lease_period=0, silent=False): os.rename(fullpath, self.fullpath) return self except Exception: - if not silent: - raise + pass # keep silent def lease(self, period): timestamp = _now() + _seconds(period) @@ -90,11 +87,11 @@ def __init__( self.maintenance_period = maintenance_period self.retention_period = retention_period self.write_timeout = write_timeout - self._maintenance_routine(silent=False) + # Run maintenance routine once upon instantiating + self._maintenance_routine() self._maintenance_task = PeriodicTask( interval=self.maintenance_period, function=self._maintenance_routine, - kwargs={'silent': True}, ) self._maintenance_task.daemon = True self._maintenance_task.start() @@ -109,19 +106,18 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.close() - def _maintenance_routine(self, silent=False): + def _maintenance_routine(self): try: if not os.path.isdir(self.path): os.makedirs(self.path) except Exception: - if not silent: - raise + # Race case will throw OSError which we can ignore + pass try: for blob in self.gets(): pass except Exception: - if not silent: - raise + pass # keep silent def gets(self): now = _now() @@ -164,7 +160,7 @@ def get(self): pass return None - def put(self, data, lease_period=0, silent=False): + def put(self, data, lease_period=0): if not self._check_storage_size(): return None blob = LocalFileBlob(os.path.join( @@ -174,7 +170,7 @@ def put(self, data, lease_period=0, silent=False): '{:08x}'.format(random.getrandbits(32)), # thread-safe random ), )) - return blob.put(data, lease_period=lease_period, silent=silent) + return blob.put(data, lease_period=lease_period) def _check_storage_size(self): size = 0 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py index 2b2d24a57..58c287b97 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py @@ -14,6 +14,7 @@ import json import logging + import requests logger = logging.getLogger(__name__) @@ -25,12 +26,12 @@ def _transmit_from_storage(self): # give a few more seconds for blob lease operation # to reduce the chance of race (for perf consideration) if blob.lease(self.options.timeout + 5): - envelopes = blob.get() # TODO: handle error + envelopes = blob.get() result = self._transmit(envelopes) if result > 0: blob.lease(result) else: - blob.delete(silent=True) + blob.delete() def _transmit(self, envelopes): """ @@ -40,6 +41,8 @@ def _transmit(self, envelopes): Return the next retry time in seconds for retryable failure. This function should never throw exception. """ + if not envelopes: + return 0 try: response = requests.post( url=self.options.endpoint, @@ -50,8 +53,13 @@ def _transmit(self, envelopes): }, timeout=self.options.timeout, ) + except requests.Timeout: + logger.warning( + 'Request time out. Ingestion may be backed up. Retrying.') + return self.options.minimum_retry_interval except Exception as ex: # TODO: consider RequestException - logger.warning('Transient client side error %s.', ex) + logger.warning( + 'Retrying due to transient client side error %s.', ex) # client side error (retryable) return self.options.minimum_retry_interval diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py index 307a019d6..4f6226f0f 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py @@ -19,6 +19,7 @@ class ProcessorTimeMetric(object): NAME = "\\Processor(_Total)\\% Processor Time" + @staticmethod def get_value(): cpu_times_percent = psutil.cpu_times_percent() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py index ccf80dda6..f24a7099d 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py @@ -19,6 +19,7 @@ class AvailableMemoryMetric(object): NAME = "\\Memory\\Available Bytes" + @staticmethod def get_value(): return psutil.virtual_memory().available diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py index 454f82f8d..75d53fe21 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py @@ -24,6 +24,7 @@ class ProcessMemoryMetric(object): NAME = "\\Process(??APP_WIN32_PROC??)\\Private Bytes" + @staticmethod def get_value(): try: diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index 716c9dc55..fc212aed8 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import shutil import unittest @@ -734,183 +733,3 @@ def test_span_data_to_envelope(self): self.assertFalse(envelope.data.baseData.success) exporter._stop() - - def test_transmission_nothing(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - with mock.patch('requests.post') as post: - post.return_value = None - exporter._transmit_from_storage() - exporter._stop() - - def test_transmission_pre_exception(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post', throw(Exception)): - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - exporter._stop() - - @mock.patch('requests.post', return_value=mock.Mock()) - def test_transmission_lease_failure(self, requests_mock): - requests_mock.return_value = MockResponse(200, 'unknown') - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('opencensus.ext.azure.common.storage.LocalFileBlob.lease') as lease: # noqa: E501 - lease.return_value = False - exporter._transmit_from_storage() - self.assertTrue(exporter.storage.get()) - exporter._stop() - - def test_transmission_exception(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(200, None) - del post.return_value.text - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_200(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(200, 'unknown') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_206(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, 'unknown') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - exporter._stop() - - def test_transmission_206_500(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3, 4, 5]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 5, - 'itemsAccepted': 3, - 'errors': [ - { - 'index': 0, - 'statusCode': 400, - 'message': '', - }, - { - 'index': 2, - 'statusCode': 500, - 'message': 'Internal Server Error', - }, - ], - })) - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - self.assertEqual(exporter.storage.get().get(), (3,)) - exporter._stop() - - def test_transmission_206_no_retry(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 3, - 'itemsAccepted': 2, - 'errors': [ - { - 'index': 0, - 'statusCode': 400, - 'message': '', - }, - ], - })) - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_206_bogus(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3, 4, 5]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 5, - 'itemsAccepted': 3, - 'errors': [ - { - 'foo': 0, - 'bar': 1, - }, - ], - })) - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_400(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(400, '{}') - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_500(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(500, '{}') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - exporter._stop() - - -class MockResponse(object): - def __init__(self, status_code, text): - self.status_code = status_code - self.text = text diff --git a/contrib/opencensus-ext-azure/tests/test_processor.py b/contrib/opencensus-ext-azure/tests/test_processor_mixin.py similarity index 100% rename from contrib/opencensus-ext-azure/tests/test_processor.py rename to contrib/opencensus-ext-azure/tests/test_processor_mixin.py diff --git a/contrib/opencensus-ext-azure/tests/test_storage.py b/contrib/opencensus-ext-azure/tests/test_storage.py index 98e1196b3..666f68647 100644 --- a/contrib/opencensus-ext-azure/tests/test_storage.py +++ b/contrib/opencensus-ext-azure/tests/test_storage.py @@ -45,39 +45,33 @@ def func(*_args, **_kwargs): class TestLocalFileBlob(unittest.TestCase): def test_delete(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - blob.delete(silent=True) - self.assertRaises(Exception, lambda: blob.delete()) - self.assertRaises(Exception, lambda: blob.delete(silent=False)) + blob.delete() + with mock.patch('os.remove') as m: + blob.delete() + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'foobar')) def test_get(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - self.assertIsNone(blob.get(silent=True)) - self.assertRaises(Exception, lambda: blob.get()) - self.assertRaises(Exception, lambda: blob.get(silent=False)) - - def test_put_error(self): - blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - with mock.patch('os.rename', side_effect=throw(Exception)): - self.assertRaises(Exception, lambda: blob.put([1, 2, 3])) + self.assertIsNone(blob.get()) def test_put_without_lease(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) input = (1, 2, 3) - blob.delete(silent=True) + blob.delete() blob.put(input) self.assertEqual(blob.get(), input) def test_put_with_lease(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) input = (1, 2, 3) - blob.delete(silent=True) + blob.delete() blob.put(input, lease_period=0.01) blob.lease(0.01) self.assertEqual(blob.get(), input) def test_lease_error(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) - blob.delete(silent=True) + blob.delete() self.assertEqual(blob.lease(0.01), None) @@ -113,8 +107,7 @@ def test_put(self): with LocalFileStorage(os.path.join(TEST_FOLDER, 'bar')) as stor: self.assertEqual(stor.get().get(), input) with mock.patch('os.rename', side_effect=throw(Exception)): - self.assertIsNone(stor.put(input, silent=True)) - self.assertRaises(Exception, lambda: stor.put(input)) + self.assertIsNone(stor.put(input)) def test_put_max_size(self): input = (1, 2, 3) @@ -158,33 +151,27 @@ def test_check_storage_size_error(self): os_mock.return_value = True self.assertTrue(stor._check_storage_size()) - def test_maintanence_routine(self): + def test_maintenance_routine(self): + with mock.patch('os.makedirs') as m: + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.makedirs') as m: m.return_value = None - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.makedirs', side_effect=throw(Exception)): - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.listdir', side_effect=throw(Exception)): - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) as stor: - with mock.patch('os.listdir', side_effect=throw(Exception)): - stor._maintenance_routine(silent=True) - self.assertRaises( - Exception, - lambda: stor._maintenance_routine(), - ) - with mock.patch('os.path.isdir', side_effect=throw(Exception)): - stor._maintenance_routine(silent=True) - self.assertRaises( - Exception, - lambda: stor._maintenance_routine(), - ) + with mock.patch('os.listdir', side_effect=throw(Exception)) as p: + stor._maintenance_routine() + stor._maintenance_routine() + self.assertEqual(p.call_count, 2) + patch = 'os.path.isdir' + with mock.patch(patch, side_effect=throw(Exception)) as isdir: + stor._maintenance_routine() + stor._maintenance_routine() + self.assertEqual(isdir.call_count, 2) diff --git a/contrib/opencensus-ext-azure/tests/test_transport_mixin.py b/contrib/opencensus-ext-azure/tests/test_transport_mixin.py new file mode 100644 index 000000000..0d1793d10 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_transport_mixin.py @@ -0,0 +1,225 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import unittest + +import mock +import requests + +from opencensus.ext.azure.common import Options +from opencensus.ext.azure.common.storage import LocalFileStorage +from opencensus.ext.azure.common.transport import TransportMixin + +TEST_FOLDER = os.path.abspath('.test.storage') + + +def setUpModule(): + os.makedirs(TEST_FOLDER) + + +def tearDownModule(): + shutil.rmtree(TEST_FOLDER) + + +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + return func + + +class MockResponse(object): + def __init__(self, status_code, text): + self.status_code = status_code + self.text = text + + +# pylint: disable=W0212 +class TestTransportMixin(unittest.TestCase): + def test_transmission_nothing(self): + mixin = TransportMixin() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + with mock.patch('requests.post') as post: + post.return_value = None + mixin._transmit_from_storage() + + def test_transmission_pre_timeout(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(requests.Timeout)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_transmission_pre_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(Exception)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_transmission_lease_failure(self, requests_mock): + requests_mock.return_value = MockResponse(200, 'unknown') + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch( + 'opencensus.ext.azure.common.storage.LocalFileBlob.lease' + ) as lease: # noqa: E501 + lease.return_value = False + mixin._transmit_from_storage() + self.assertTrue(mixin.storage.get()) + + def test_transmission_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, None) + del post.return_value.text + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_200(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_206(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_transmission_206_500(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3, 4, 5]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, + 'message': '', + }, + { + 'index': 2, + 'statusCode': 500, + 'message': 'Internal Server Error', + }, + ], + })) + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + self.assertEqual(mixin.storage.get().get(), (3,)) + + def test_transmission_206_no_retry(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 3, + 'itemsAccepted': 2, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, + 'message': '', + }, + ], + })) + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_206_bogus(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3, 4, 5]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'foo': 0, + 'bar': 1, + }, + ], + })) + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_400(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(400, '{}') + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_500(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(500, '{}') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) diff --git a/tox.ini b/tox.ini index 2d16b27b2..b5396cb63 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ commands = ; TODO system tests lint: isort --check-only --diff --recursive . lint: flake8 context/ contrib/ opencensus/ tests/ examples/ - lint: - bash ./scripts/pylint.sh + ; lint: - bash ./scripts/pylint.sh py37-setup: python setup.py check --restructuredtext --strict py37-docs: bash ./scripts/update_docs.sh ; TODO deployment From d79c1fa7890f15ac5e550edf42b88814bed42ef0 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 17 Jun 2020 00:07:15 -0700 Subject: [PATCH 4/7] Add support to initialize azure exporters with proxies --- contrib/opencensus-ext-azure/CHANGELOG.md | 2 ++ .../opencensus/ext/azure/common/__init__.py | 5 ++++- .../opencensus/ext/azure/common/transport.py | 1 + .../tests/test_azure_log_exporter.py | 11 +++++++++++ .../tests/test_azure_trace_exporter.py | 11 +++++++++++ .../opencensus-ext-azure/tests/test_options.py | 18 ++++++++++++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index fcee072cb..623ce5662 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -4,6 +4,8 @@ - Change default path of local storage ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) +- Add support to initialize azure exporters with proxies + ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) ## 1.0.1 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index fcb25acb5..8d76d91ea 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -56,6 +56,9 @@ def process_options(options): TEMPDIR_PREFIX + TEMPDIR_SUFFIX ) + if options.proxies is None: + options.proxies = '{}' + def parse_connection_string(connection_string): if connection_string is None: @@ -105,7 +108,7 @@ def __init__(self, *args, **kwargs): logging_sampling_rate=1.0, max_batch_size=100, minimum_retry_interval=60, # minimum retry interval in seconds - proxy=None, + proxies=None, # string maps url schemes to the url of the proxies storage_maintenance_period=60, storage_max_size=50*1024*1024, # 50MiB storage_path=None, diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py index 58c287b97..3643da02b 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py @@ -52,6 +52,7 @@ def _transmit(self, envelopes): 'Content-Type': 'application/json; charset=utf-8', }, timeout=self.options.timeout, + proxies=json.loads(self.options.proxies), ) except requests.Timeout: logger.warning( diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 8aa6baa8d..88b8c436b 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -85,6 +85,17 @@ def test_invalid_sampling_rate(self): logging_sampling_rate=4.0, ) + def test_init_handler_with_proxies(self): + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + handler.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index fc212aed8..5ff70e234 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -45,6 +45,17 @@ def test_ctor(self): self.assertRaises(ValueError, lambda: trace_exporter.AzureExporter()) Options._default.instrumentation_key = instrumentation_key + def test_init_exporter_with_proxies(self): + exporter = trace_exporter.AzureExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + exporter.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_emit_empty(self, request_mock): exporter = trace_exporter.AzureExporter( diff --git a/contrib/opencensus-ext-azure/tests/test_options.py b/contrib/opencensus-ext-azure/tests/test_options.py index 5c16d9c6c..969eb5a89 100644 --- a/contrib/opencensus-ext-azure/tests/test_options.py +++ b/contrib/opencensus-ext-azure/tests/test_options.py @@ -90,6 +90,24 @@ def test_process_options_endpoint_default(self): self.assertEqual(options.endpoint, 'https://dc.services.visualstudio.com/v2/track') + def test_process_options_proxies_default(self): + options = common.Options() + options.proxies = "{}" + common.process_options(options) + + self.assertEqual(options.proxies, "{}") + + def test_process_options_proxies_set_proxies(self): + options = common.Options() + options.connection_string = None + options.proxies = '{"https": "https://test-proxy.com"}' + common.process_options(options) + + self.assertEqual( + options.proxies, + '{"https": "https://test-proxy.com"}' + ) + def test_parse_connection_string_none(self): cs = None result = common.parse_connection_string(cs) From 1c54b3f79d4de3ee33514d04e0cfb5d653b9c819 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 17 Jun 2020 00:09:37 -0700 Subject: [PATCH 5/7] fix is_exporter context --- .../opencensus/ext/azure/common/exporter.py | 1 + .../opencensus/ext/azure/log_exporter/__init__.py | 2 +- opencensus/common/transports/async_.py | 6 +++--- opencensus/common/transports/sync.py | 5 +++++ opencensus/metrics/transport.py | 1 + 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py index b6eaed7e6..55639a85f 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py @@ -66,6 +66,7 @@ def __init__(self, src, dst): def run(self): # pragma: NO COVER # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread. execution_context.set_is_exporter(True) src = self.src dst = self.dst diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py index 6bc4db53f..c8bc6e84f 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py @@ -79,7 +79,7 @@ def __init__(self, src, dst): def run(self): # Indicate that this thread is an exporter thread. - execution_context.set_is_exporter(True) + # Used to suppress tracking of requests in this thread. src = self._src dst = self._dst while True: diff --git a/opencensus/common/transports/async_.py b/opencensus/common/transports/async_.py index 61f3ea020..cc825d81f 100644 --- a/opencensus/common/transports/async_.py +++ b/opencensus/common/transports/async_.py @@ -93,6 +93,9 @@ def _thread_main(self): Pulls pending data off the queue and writes them in batches to the specified tracing backend using the exporter. """ + # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread. + execution_context.set_is_exporter(True) quit_ = False while True: @@ -142,9 +145,6 @@ def start(self): self._thread = threading.Thread( target=self._thread_main, name=_WORKER_THREAD_NAME) self._thread.daemon = True - # Indicate that this thread is an exporter thread. Used for - # auto-collection. - execution_context.set_is_exporter(True) self._thread.start() atexit.register(self._export_pending_data) diff --git a/opencensus/common/transports/sync.py b/opencensus/common/transports/sync.py index aae01218f..bffdbf21a 100644 --- a/opencensus/common/transports/sync.py +++ b/opencensus/common/transports/sync.py @@ -20,4 +20,9 @@ def __init__(self, exporter): self.exporter = exporter def export(self, datas): + # Used to suppress tracking of requests in export + execution_context.set_is_exporter(True) self.exporter.emit(datas) + # Reset the context + execution_context.set_is_exporter(False) + diff --git a/opencensus/metrics/transport.py b/opencensus/metrics/transport.py index 02579b279..61faf2b42 100644 --- a/opencensus/metrics/transport.py +++ b/opencensus/metrics/transport.py @@ -67,6 +67,7 @@ def func(*aa, **kw): def run(self): # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread. execution_context.set_is_exporter(True) super(PeriodicMetricTask, self).run() From 0e4b39051cdd02f981b4af044993d96498355067 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 17 Jun 2020 08:44:41 -0700 Subject: [PATCH 6/7] changelog --- CHANGELOG.md | 8 ++++++++ contrib/opencensus-ext-azure/CHANGELOG.md | 13 +++++++++++++ .../opencensus/ext/azure/common/version.py | 2 +- opencensus/common/version/__init__.py | 2 +- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f13147ba8..19f2d84b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,16 @@ ## Unreleased +## 0.7.8 +Released 2020-06-17 + +- Updated `azure` module + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), + [#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) + ## 0.7.7 Released 2020-02-04 + - Updated `azure` module ([#837](https://github.com/census-instrumentation/opencensus-python/pull/837), [#845](https://github.com/census-instrumentation/opencensus-python/pull/845), diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 623ce5662..6d26113dc 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,11 +2,24 @@ ## Unreleased +## 1.0.3 +Released 2020-06-17 + - Change default path of local storage ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) - Add support to initialize azure exporters with proxies ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) +## 1.0.2 +Released 2020-02-04 + +- Add local storage and retry logic for Azure Metrics Exporter + ([#845](https://github.com/census-instrumentation/opencensus-python/pull/845)) +- Add Fixed-rate sampling logic for Azure Log Exporter + ([#848](https://github.com/census-instrumentation/opencensus-python/pull/848)) +- Implement TelemetryProcessors for Azure exporters + ([#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) + ## 1.0.1 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py index 4b19db0b6..8ef965cab 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '1.0.2' +__version__ = '1.0.3' diff --git a/opencensus/common/version/__init__.py b/opencensus/common/version/__init__.py index d16aadd71..648eaf310 100644 --- a/opencensus/common/version/__init__.py +++ b/opencensus/common/version/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.7' +__version__ = '0.7.8' From 1c3c6cfa2ea11607af04dc490cee6d43c07fb7ec Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 17 Jun 2020 09:25:22 -0700 Subject: [PATCH 7/7] triger build --- contrib/opencensus-ext-azure/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 6d26113dc..7efa8b779 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -10,6 +10,7 @@ Released 2020-06-17 - Add support to initialize azure exporters with proxies ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) + ## 1.0.2 Released 2020-02-04