0.24.0

.gitignore

@@ -41,7 +41,6 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
-nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/

README.md

@@ -46,29 +46,9 @@ launch them through:
 [docker-compose]: https://www.docker.com/products/docker-compose
 
 
-#### Running the Tests in your local environment
-
-Once docker is up and running you should be able to run the tests. To launch a
-single test manually. For example to run the tests for `redis-py` 2.10 on Python
-3.5 and 3.6:
-
-    $ tox -e '{py35,py36}-redis{210}'
-
-To see the defined test commands see `tox.ini`.
-
-To launch the complete test matrix run:
-
-    $ tox
-
-
 #### Running Tests in docker
 
-If you prefer not to setup your local machine to run tests, we provide a preconfigured docker image.
-Note that this image is the same used in CircleCI to run tests.
-
-You still need docker containers running additional services up and running.
-
-Run the test runner
+Once your docker-compose environment is running, you can run the test runner image:
 
     $ docker-compose run --rm testrunner
 

Rakefile

@@ -12,13 +12,9 @@ S3_BUCKET = "pypi.datadoghq.com"
 
 desc "release the a new wheel"
 task :'release:wheel' do
-  # Use mkwheelhouse to build the wheel, push it to S3 then update the repo index
-  # If at some point, we need only the 2 first steps:
-  #  - python setup.py bdist_wheel
-  #  - aws s3 cp dist/*.whl s3://pypi.datadoghq.com/#{s3_dir}/
   fail "Missing environment variable S3_DIR" if !S3_DIR or S3_DIR.empty?
 
-  # Use custom mkwheelhouse script to build and upload an sdist to S3 bucket
+  # Use custom `mkwheelhouse` to upload wheels and source distribution from dist/ to S3 bucket
   sh "scripts/mkwheelhouse"
 end
 
@@ -66,6 +62,7 @@ namespace :pypi do
 
   task :build => :clean do
     puts "building release in #{RELEASE_DIR}"
+    # TODO: Use `scripts/build-dist` instead to build sdist and wheels
     sh "python setup.py -q sdist -d #{RELEASE_DIR}"
   end
 

ddtrace/__init__.py

@@ -4,7 +4,7 @@
 from .tracer import Tracer
 from .settings import config
 
-__version__ = '0.23.0'
+__version__ = '0.24.0'
 
 # a global tracer instance with integration settings
 tracer = Tracer()

ddtrace/bootstrap/sitecustomize.py

@@ -46,7 +46,10 @@
 
 
 def update_patched_modules():
-    for patch in os.environ.get("DATADOG_PATCH_MODULES", '').split(','):
+    modules_to_patch = os.environ.get("DATADOG_PATCH_MODULES")
+    if not modules_to_patch:
+        return
+    for patch in modules_to_patch.split(','):
         if len(patch.split(':')) != 2:
             log.debug("skipping malformed patch instruction")
             continue
@@ -95,6 +98,8 @@ def add_global_tags(tracer):
     if priority_sampling:
         opts["priority_sampling"] = asbool(priority_sampling)
 
+    opts['collect_metrics'] = asbool(get_env('runtime_metrics', 'enabled'))
+
     if opts:
         tracer.configure(**opts)
 

ddtrace/commands/ddtrace_run.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
-
 from distutils import spawn
 import os
 import sys

ddtrace/contrib/pyramid/patch.py

@@ -78,7 +78,7 @@ def insert_tween_if_needed(settings):
     # pyramid.  We need our tween to be before it, otherwise unhandled
     # exceptions will be caught before they reach our tween.
     idx = tweens.find(pyramid.tweens.EXCVIEW)
-    if idx is -1:
+    if idx == -1:
         settings['pyramid.tweens'] = tweens + '\n' + DD_TWEEN_NAME
     else:
         settings['pyramid.tweens'] = tweens[:idx] + DD_TWEEN_NAME + "\n" + tweens[idx:]

ddtrace/contrib/tornado/__init__.py

@@ -58,7 +58,7 @@ def notify(self):
             'analytics_enabled': False,
             'settings': {
                 'FILTERS':  [
-                    FilterRequestsOnUrl(r'http://test\.example\.com'),
+                    FilterRequestsOnUrl(r'http://test\\.example\\.com'),
                 ],
             },
         },

ddtrace/filters.py

@@ -18,15 +18,15 @@ class FilterRequestsOnUrl(object):
 
     To filter out http calls to domain api.example.com::
 
-        FilterRequestsOnUrl(r'http://api\.example\.com')
+        FilterRequestsOnUrl(r'http://api\\.example\\.com')
 
     To filter out http calls to all first level subdomains from example.com::
 
-        FilterRequestOnUrl(r'http://.*+\.example\.com')
+        FilterRequestOnUrl(r'http://.*+\\.example\\.com')
 
     To filter out calls to both http://test.example.com and http://example.com/healthcheck::
 
-        FilterRequestOnUrl([r'http://test\.example\.com', r'http://example\.com/healthcheck'])
+        FilterRequestOnUrl([r'http://test\\.example\\.com', r'http://example\\.com/healthcheck'])
     """
     def __init__(self, regexps):
         if isinstance(regexps, str):

ddtrace/helpers.py

@@ -32,6 +32,6 @@ def get_correlation_ids(tracer=None):
         return None, None
 
     span = tracer.current_span()
-    if span is None:
+    if not span:
         return None, None
     return span.trace_id, span.span_id

ddtrace/internal/runtime/__init__.py

@@ -0,0 +1,12 @@
+from .runtime_metrics import (
+    RuntimeTags,
+    RuntimeMetrics,
+    RuntimeWorker,
+)
+
+
+__all__ = [
+    'RuntimeTags',
+    'RuntimeMetrics',
+    'RuntimeWorker',
+]

ddtrace/internal/runtime/collector.py

@@ -0,0 +1,85 @@
+import importlib
+
+from ..logger import get_logger
+
+log = get_logger(__name__)
+
+
+class ValueCollector(object):
+    """A basic state machine useful for collecting, caching and updating data
+    obtained from different Python modules.
+
+    The two primary use-cases are
+    1) data loaded once (like tagging information)
+    2) periodically updating data sources (like thread count)
+
+    Functionality is provided for requiring and importing modules which may or
+    may not be installed.
+    """
+    enabled = True
+    periodic = False
+    required_modules = []
+    value = None
+    value_loaded = False
+
+    def __init__(self, enabled=None, periodic=None, required_modules=None):
+        self.enabled = self.enabled if enabled is None else enabled
+        self.periodic = self.periodic if periodic is None else periodic
+        self.required_modules = self.required_modules if required_modules is None else required_modules
+
+        self._modules_successfully_loaded = False
+        self.modules = self._load_modules()
+        if self._modules_successfully_loaded:
+            self._on_modules_load()
+
+    def _on_modules_load(self):
+        """Hook triggered after all required_modules have been successfully loaded.
+        """
+
+    def _load_modules(self):
+        modules = {}
+        try:
+            for module in self.required_modules:
+                modules[module] = importlib.import_module(module)
+            self._modules_successfully_loaded = True
+        except ImportError:
+            # DEV: disable collector if we cannot load any of the required modules
+            self.enabled = False
+            log.warn('Could not import module "{}" for {}. Disabling collector.'.format(module, self))
+            return None
+        return modules
+
+    def collect(self, keys=None):
+        """Returns metrics as collected by `collect_fn`.
+
+        :param keys: The keys of the metrics to collect.
+        """
+        if not self.enabled:
+            return self.value
+
+        keys = keys or set()
+
+        if not self.periodic and self.value_loaded:
+            return self.value
+
+        # call underlying collect function and filter out keys not requested
+        self.value = self.collect_fn(keys)
+
+        # filter values for keys
+        if len(keys) > 0 and isinstance(self.value, list):
+            self.value = [
+                (k, v)
+                for (k, v) in self.value
+                if k in keys
+            ]
+
+        self.value_loaded = True
+        return self.value
+
+    def __repr__(self):
+        return '<{}(enabled={},periodic={},required_modules={})>'.format(
+            self.__class__.__name__,
+            self.enabled,
+            self.periodic,
+            self.required_modules,
+        )

ddtrace/internal/runtime/constants.py

@@ -0,0 +1,46 @@
+GC_COUNT_GEN0 = 'runtime.python.gc.count.gen0'
+GC_COUNT_GEN1 = 'runtime.python.gc.count.gen1'
+GC_COUNT_GEN2 = 'runtime.python.gc.count.gen2'
+
+THREAD_COUNT = 'runtime.python.thread_count'
+MEM_RSS = 'runtime.python.mem.rss'
+CPU_TIME_SYS = 'runtime.python.cpu.time.sys'
+CPU_TIME_USER = 'runtime.python.cpu.time.user'
+CPU_PERCENT = 'runtime.python.cpu.percent'
+CTX_SWITCH_VOLUNTARY = 'runtime.python.cpu.ctx_switch.voluntary'
+CTX_SWITCH_INVOLUNTARY = 'runtime.python.cpu.ctx_switch.involuntary'
+
+GC_RUNTIME_METRICS = set([
+    GC_COUNT_GEN0,
+    GC_COUNT_GEN1,
+    GC_COUNT_GEN2,
+])
+
+PSUTIL_RUNTIME_METRICS = set([
+    THREAD_COUNT,
+    MEM_RSS,
+    CTX_SWITCH_VOLUNTARY,
+    CTX_SWITCH_INVOLUNTARY,
+    CPU_TIME_SYS,
+    CPU_TIME_USER,
+    CPU_PERCENT,
+])
+
+DEFAULT_RUNTIME_METRICS = GC_RUNTIME_METRICS | PSUTIL_RUNTIME_METRICS
+
+RUNTIME_ID = 'runtime-id'
+SERVICE = 'service'
+LANG_INTERPRETER = 'lang_interpreter'
+LANG_VERSION = 'lang_version'
+
+TRACER_TAGS = set([
+    RUNTIME_ID,
+    SERVICE,
+])
+
+PLATFORM_TAGS = set([
+    LANG_INTERPRETER,
+    LANG_VERSION
+])
+
+DEFAULT_RUNTIME_TAGS = TRACER_TAGS

ddtrace/internal/runtime/metric_collectors.py

@@ -0,0 +1,92 @@
+import os
+
+from .collector import ValueCollector
+from .constants import (
+    GC_COUNT_GEN0,
+    GC_COUNT_GEN1,
+    GC_COUNT_GEN2,
+    THREAD_COUNT,
+    MEM_RSS,
+    CTX_SWITCH_VOLUNTARY,
+    CTX_SWITCH_INVOLUNTARY,
+    CPU_TIME_SYS,
+    CPU_TIME_USER,
+    CPU_PERCENT,
+)
+
+
+class RuntimeMetricCollector(ValueCollector):
+    value = []
+    periodic = True
+
+
+class GCRuntimeMetricCollector(RuntimeMetricCollector):
+    """ Collector for garbage collection generational counts
+
+    More information at https://docs.python.org/3/library/gc.html
+    """
+    required_modules = ['gc']
+
+    def collect_fn(self, keys):
+        gc = self.modules.get('gc')
+
+        counts = gc.get_count()
+        metrics = [
+            (GC_COUNT_GEN0, counts[0]),
+            (GC_COUNT_GEN1, counts[1]),
+            (GC_COUNT_GEN2, counts[2]),
+        ]
+
+        return metrics
+
+
+class PSUtilRuntimeMetricCollector(RuntimeMetricCollector):
+    """Collector for psutil metrics.
+
+    Performs batched operations via proc.oneshot() to optimize the calls.
+    See https://psutil.readthedocs.io/en/latest/#psutil.Process.oneshot
+    for more information.
+    """
+    required_modules = ['psutil']
+    stored_value = dict(
+        CPU_TIME_SYS_TOTAL=0,
+        CPU_TIME_USER_TOTAL=0,
+        CTX_SWITCH_VOLUNTARY_TOTAL=0,
+        CTX_SWITCH_INVOLUNTARY_TOTAL=0,
+    )
+
+    def _on_modules_load(self):
+        self.proc = self.modules['psutil'].Process(os.getpid())
+
+    def collect_fn(self, keys):
+        with self.proc.oneshot():
+            # only return time deltas
+            # TODO[tahir]: better abstraction for metrics based on last value
+            cpu_time_sys_total = self.proc.cpu_times().system
+            cpu_time_user_total = self.proc.cpu_times().user
+            cpu_time_sys = cpu_time_sys_total - self.stored_value['CPU_TIME_SYS_TOTAL']
+            cpu_time_user = cpu_time_user_total - self.stored_value['CPU_TIME_USER_TOTAL']
+
+            ctx_switch_voluntary_total = self.proc.num_ctx_switches().voluntary
+            ctx_switch_involuntary_total = self.proc.num_ctx_switches().involuntary
+            ctx_switch_voluntary = ctx_switch_voluntary_total - self.stored_value['CTX_SWITCH_VOLUNTARY_TOTAL']
+            ctx_switch_involuntary = ctx_switch_involuntary_total - self.stored_value['CTX_SWITCH_INVOLUNTARY_TOTAL']
+
+            self.stored_value = dict(
+                CPU_TIME_SYS_TOTAL=cpu_time_sys_total,
+                CPU_TIME_USER_TOTAL=cpu_time_user_total,
+                CTX_SWITCH_VOLUNTARY_TOTAL=ctx_switch_voluntary_total,
+                CTX_SWITCH_INVOLUNTARY_TOTAL=ctx_switch_involuntary_total,
+            )
+
+            metrics = [
+                (THREAD_COUNT, self.proc.num_threads()),
+                (MEM_RSS, self.proc.memory_info().rss),
+                (CTX_SWITCH_VOLUNTARY, ctx_switch_voluntary),
+                (CTX_SWITCH_INVOLUNTARY, ctx_switch_involuntary),
+                (CPU_TIME_SYS, cpu_time_sys),
+                (CPU_TIME_USER, cpu_time_user),
+                (CPU_PERCENT, self.proc.cpu_percent()),
+            ]
+
+            return metrics