diff --git a/Appraisals b/Appraisals index b9deaf3eb36..106a03b2c55 100644 --- a/Appraisals +++ b/Appraisals @@ -182,7 +182,7 @@ elsif Gem::Version.new('2.1.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'delayed_job' gem 'delayed_job_active_record' gem 'elasticsearch-transport' - gem 'presto-client', '>= 0.5.14' + gem 'presto-client', '>= 0.5.14' gem 'ethon' gem 'excon' gem 'hiredis' @@ -369,6 +369,7 @@ elsif Gem::Version.new('2.2.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'sqlite3', '~> 1.3.6' gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end end elsif Gem::Version.new('2.3.0') <= Gem::Version.new(RUBY_VERSION) \ @@ -533,6 +534,7 @@ elsif Gem::Version.new('2.3.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'sqlite3', '~> 1.3.6' gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end appraise 'contrib-old' do @@ -616,6 +618,7 @@ elsif Gem::Version.new('2.4.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'sqlite3', '~> 1.3.6' gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end appraise 'contrib-old' do @@ -743,6 +746,7 @@ elsif Gem::Version.new('2.5.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'sqlite3', '~> 1.4.1', platform: :ruby gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end appraise 'contrib-old' do @@ -858,6 +862,7 @@ elsif Gem::Version.new('2.6.0') <= Gem::Version.new(RUBY_VERSION) \ gem 'sqlite3', '~> 1.4.1' gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end appraise 'contrib-old' do @@ -976,6 +981,7 @@ elsif Gem::Version.new('2.7.0') <= Gem::Version.new(RUBY_VERSION) gem 'sqlite3', '~> 1.4.1' gem 'sucker_punch' gem 'typhoeus' + gem 'que', '>= 1.0.0.beta2' end appraise 'contrib-old' do diff --git a/Rakefile b/Rakefile index 6ffb6c9ef2b..1c319836762 100644 --- a/Rakefile +++ b/Rakefile @@ -100,6 +100,7 @@ namespace :spec do :mongodb, :mysql2, :presto, + :que, :racecar, :rack, :rake, @@ -339,6 +340,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' @@ -413,6 +415,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' @@ -492,6 +495,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' @@ -553,6 +557,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' if RUBY_PLATFORM != 'java' # built-in jdbc is used instead sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' @@ -626,6 +631,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' @@ -699,6 +705,7 @@ task :ci do sh 'bundle exec appraisal contrib rake spec:mongodb' sh 'bundle exec appraisal contrib rake spec:mysql2' sh 'bundle exec appraisal contrib rake spec:presto' + sh 'bundle exec appraisal contrib rake spec:que' sh 'bundle exec appraisal contrib rake spec:racecar' sh 'bundle exec appraisal contrib rake spec:rack' sh 'bundle exec appraisal contrib rake spec:rake' diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 6535f6053b2..6e5c6cebe28 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -48,6 +48,7 @@ To contribute, check out the [contribution guidelines][contribution docs] and [d - [MySQL2](#mysql2) - [Net/HTTP](#net-http) - [Presto](#presto) + - [Que](#que) - [Racecar](#racecar) - [Rack](#rack) - [Rails](#rails) @@ -353,6 +354,7 @@ For a list of available integrations, and their configuration options, please re | MySQL2 | `mysql2` | `>= 0.3.21` | *gem not available* | *[Link](#mysql2)* | *[Link](https://github.com/brianmario/mysql2)* | | Net/HTTP | `http` | *(Any supported Ruby)* | *(Any supported Ruby)* | *[Link](#nethttp)* | *[Link](https://ruby-doc.org/stdlib-2.4.0/libdoc/net/http/rdoc/Net/HTTP.html)* | | Presto | `presto` | `>= 0.5.14` | `>= 0.5.14` | *[Link](#presto)* | *[Link](https://github.com/treasure-data/presto-client-ruby)* | +| Que | `que` | `>= 1.0.0.beta2` | `>= 1.0.0.beta2` | *[Link](#que)* | *[Link](https://github.com/que-rb/que)* | | Racecar | `racecar` | `>= 0.3.5` | `>= 0.3.5` | *[Link](#racecar)* | *[Link](https://github.com/zendesk/racecar)* | | Rack | `rack` | `>= 1.1` | `>= 1.1` | *[Link](#rack)* | *[Link](https://github.com/rack/rack)* | | Rails | `rails` | `>= 3.0` | `>= 3.0` | *[Link](#rails)* | *[Link](https://github.com/rails/rails)* | @@ -1111,6 +1113,30 @@ Where `options` is an optional `Hash` that accepts the following parameters: | `analytics_enabled` | Enable analytics for spans produced by this integration. `true` for on, `nil` to defer to global setting, `false` for off. | `false` | | `service_name` | Service name used for `presto` instrumentation | `'presto'` | +### Que + +The Que integration is a middleware which will trace job executions. + +You can enable it through `Datadog.configure`: + +```ruby +require 'ddtrace' + +Datadog.configure do |c| + c.use :que, options +end +``` + +Where `options` is an optional `Hash` that accepts the following parameters: + +| Key | Description | Default | +| --- | ----------- | ------- | +| `analytics_enabled` | Enable analytics for spans produced by this integration. `true` for on, `nil` to defer to global setting, `false` for off. | `false` | +| `enabled` | Defines whether Que should be traced. Useful for temporarily disabling tracing. `true` or `false` | `true` | +| `service_name` | Service name used for `que` instrumentation | `'que'` | +| `tag_args` | Enable tagging of a job's args field. `true` for on, `false` for off. | `false` | +| `tag_data` | Enable tagging of a job's data field. `true` for on, `false` for off. | `false` | + ### Racecar The Racecar integration provides tracing for Racecar jobs. diff --git a/lib/ddtrace.rb b/lib/ddtrace.rb index b537e4bba7f..eb5a648c68a 100644 --- a/lib/ddtrace.rb +++ b/lib/ddtrace.rb @@ -59,6 +59,7 @@ module Datadog require 'ddtrace/contrib/integration' require 'ddtrace/contrib/kafka/integration' require 'ddtrace/contrib/presto/integration' +require 'ddtrace/contrib/que/integration' require 'ddtrace/contrib/mysql2/integration' require 'ddtrace/contrib/mongodb/integration' require 'ddtrace/contrib/racecar/integration' diff --git a/lib/ddtrace/contrib/que/configuration/settings.rb b/lib/ddtrace/contrib/que/configuration/settings.rb new file mode 100644 index 00000000000..100db3970da --- /dev/null +++ b/lib/ddtrace/contrib/que/configuration/settings.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'ddtrace/contrib/configuration/settings' + +module Datadog + module Contrib + module Que + module Configuration + # Default settings for the Que integration + class Settings < Datadog::Contrib::Configuration::Settings + option :service_name, default: Ext::SERVICE_NAME + option :distributed_tracing, default: true + + option :enabled do |o| + o.default { env_to_bool(Ext::ENV_ENABLED, true) } + o.lazy + end + + option :analytics_enabled do |o| + o.default { env_to_bool([Ext::ENV_ANALYTICS_ENABLED, Ext::ENV_ANALYTICS_ENABLED_OLD], false) } + o.lazy + end + + option :analytics_sample_rate do |o| + o.default { env_to_float([Ext::ENV_ANALYTICS_SAMPLE_RATE, Ext::ENV_ANALYTICS_SAMPLE_RATE_OLD], 1.0) } + o.lazy + end + + option :tag_args do |o| + o.default { env_to_bool(Ext::ENV_TAG_ARGS_ENABLED, false) } + o.lazy + end + + option :tag_data do |o| + o.default { env_to_bool(Ext::ENV_TAG_DATA_ENABLED, false) } + o.lazy + end + end + end + end + end +end diff --git a/lib/ddtrace/contrib/que/ext.rb b/lib/ddtrace/contrib/que/ext.rb new file mode 100644 index 00000000000..66b9640edef --- /dev/null +++ b/lib/ddtrace/contrib/que/ext.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Datadog + module Contrib + module Que + # Que integration constants + module Ext + APP = 'que'.freeze + ENV_ANALYTICS_ENABLED = 'DD_TRACE_QUE_ANALYTICS_ENABLED'.freeze + ENV_ANALYTICS_ENABLED_OLD = 'DD_QUE_ANALYTICS_ENABLED'.freeze + ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_QUE_ANALYTICS_SAMPLE_RATE'.freeze + ENV_ANALYTICS_SAMPLE_RATE_OLD = 'DD_QUE_ANALYTICS_SAMPLE_RATE'.freeze + ENV_ENABLED = 'DD_TRACE_QUE_ENABLED'.freeze + ENV_TAG_ARGS_ENABLED = 'DD_TRACE_QUE_TAG_ARGS_ENABLED'.freeze + ENV_TAG_DATA_ENABLED = 'DD_TRACE_QUE_TAG_DATA_ENABLED'.freeze + SERVICE_NAME = 'que'.freeze + SPAN_JOB = 'que.job'.freeze + TAG_JOB_ARGS = 'que.job.args'.freeze + TAG_JOB_DATA = 'que.job.data'.freeze + TAG_JOB_ERROR_COUNT = 'que.job.error_count'.freeze + TAG_JOB_EXPIRED_AT = 'que.job.expired_at'.freeze + TAG_JOB_FINISHED_AT = 'que.job.finished_at'.freeze + TAG_JOB_ID = 'que.job.id'.freeze + TAG_JOB_PRIORITY = 'que.job.priority'.freeze + TAG_JOB_QUEUE = 'que.job.queue'.freeze + TAG_JOB_RUN_AT = 'que.job.run_at'.freeze + end + end + end +end diff --git a/lib/ddtrace/contrib/que/integration.rb b/lib/ddtrace/contrib/que/integration.rb new file mode 100644 index 00000000000..987e940a39e --- /dev/null +++ b/lib/ddtrace/contrib/que/integration.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'ddtrace/contrib/integration' +require 'ddtrace/contrib/que/ext' +require 'ddtrace/contrib/que/configuration/settings' +require 'ddtrace/contrib/que/patcher' + +module Datadog + module Contrib + module Que + # Description of Que integration + class Integration + include Datadog::Contrib::Integration + + MINIMUM_VERSION = Gem::Version.new('1.0.0.beta2') + + register_as :que, auto_patch: true + + def self.version + Gem.loaded_specs[Datadog::Contrib::Que::Ext::APP] && + Gem.loaded_specs[Datadog::Contrib::Que::Ext::APP].version + end + + def self.loaded? + !defined?(::Que).nil? + end + + def self.compatible? + super && version >= MINIMUM_VERSION + end + + def default_configuration + Configuration::Settings.new + end + + def patcher + Patcher + end + end + end + end +end diff --git a/lib/ddtrace/contrib/que/patcher.rb b/lib/ddtrace/contrib/que/patcher.rb new file mode 100644 index 00000000000..ce81e8c06aa --- /dev/null +++ b/lib/ddtrace/contrib/que/patcher.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'ddtrace/contrib/que/tracer' + +module Datadog + module Contrib + module Que + # Patcher enables patching of 'que' module. + module Patcher + include Datadog::Contrib::Patcher + + module_function + + def target_version + Integration.version + end + + def patch + ::Que.job_middleware.push(Que::Tracer.new) + end + end + end + end +end diff --git a/lib/ddtrace/contrib/que/tracer.rb b/lib/ddtrace/contrib/que/tracer.rb new file mode 100644 index 00000000000..f25b45523e3 --- /dev/null +++ b/lib/ddtrace/contrib/que/tracer.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'ddtrace/contrib/analytics' + +module Datadog + module Contrib + module Que + # Tracer is a Que's server-side middleware which traces executed jobs + class Tracer + def call(job) + trace_options = { + service: configuration[:service_name], + span_type: Datadog::Ext::AppTypes::WORKER + } + + tracer.trace(Ext::SPAN_JOB, trace_options) do |request_span| + request_span.resource = job.class.name.to_s + request_span.set_tag(Ext::TAG_JOB_QUEUE, job.que_attrs[:queue]) + request_span.set_tag(Ext::TAG_JOB_ID, job.que_attrs[:id]) + request_span.set_tag(Ext::TAG_JOB_PRIORITY, job.que_attrs[:priority]) + request_span.set_tag(Ext::TAG_JOB_ERROR_COUNT, job.que_attrs[:error_count]) + request_span.set_tag(Ext::TAG_JOB_RUN_AT, job.que_attrs[:run_at]) + request_span.set_tag(Ext::TAG_JOB_EXPIRED_AT, job.que_attrs[:expired_at]) + request_span.set_tag(Ext::TAG_JOB_FINISHED_AT, job.que_attrs[:finished_at]) + request_span.set_tag(Ext::TAG_JOB_ARGS, job.que_attrs[:args]) if configuration[:tag_args] + request_span.set_tag(Ext::TAG_JOB_DATA, job.que_attrs[:data]) if configuration[:tag_data] + + set_sample_rate(request_span) + Contrib::Analytics.set_measured(request_span) + + yield + end + end + + private + + def set_sample_rate(request_span) + if Contrib::Analytics.enabled?(configuration[:analytics_enabled]) + Contrib::Analytics.set_sample_rate( + request_span, + configuration[:analytics_sample_rate] + ) + end + end + + def tracer + configuration[:tracer] + end + + def configuration + Datadog.configuration[Datadog::Contrib::Que::Ext::APP.to_sym] + end + end + end + end +end diff --git a/spec/ddtrace/contrib/que/integration_spec.rb b/spec/ddtrace/contrib/que/integration_spec.rb new file mode 100644 index 00000000000..6be6f054fa8 --- /dev/null +++ b/spec/ddtrace/contrib/que/integration_spec.rb @@ -0,0 +1,67 @@ +require 'ddtrace/contrib/support/spec_helper' +require 'ddtrace/contrib/que/integration' + +RSpec.describe Datadog::Contrib::Que::Integration do + extend ConfigurationHelpers + + let(:integration) { described_class.new(:que) } + + describe '.version' do + subject(:version) { described_class.version } + + context 'when the "que" gem is loaded' do + include_context 'loaded gems', que: described_class::MINIMUM_VERSION + it { is_expected.to be_a_kind_of(Gem::Version) } + end + + context 'when "que" gem is not loaded' do + include_context 'loaded gems', que: nil + it { is_expected.to be nil } + end + end + + describe '.loaded?' do + subject(:loaded?) { described_class.loaded? } + + context 'when Que is defined' do + before { stub_const('Que', Class.new) } + it { is_expected.to be true } + end + + context 'when Que is not defined' do + before { hide_const('Que') } + it { is_expected.to be false } + end + end + + describe '.compatible?' do + subject(:compatible?) { described_class.compatible? } + + context 'when "que" gem is loaded with a version' do + context 'that is less than the minimum' do + include_context 'loaded gems', que: decrement_gem_version(described_class::MINIMUM_VERSION) + it { is_expected.to be false } + end + + context 'that meets the minimum version' do + include_context 'loaded gems', que: described_class::MINIMUM_VERSION + it { is_expected.to be true } + end + end + + context 'when gem is not loaded' do + include_context 'loaded gems', que: nil + it { is_expected.to be false } + end + end + + describe '#default_configuration' do + subject(:default_configuration) { integration.default_configuration } + it { is_expected.to be_a_kind_of(Datadog::Contrib::Que::Configuration::Settings) } + end + + describe '#patcher' do + subject(:patcher) { integration.patcher } + it { is_expected.to be Datadog::Contrib::Que::Patcher } + end +end diff --git a/spec/ddtrace/contrib/que/patcher_spec.rb b/spec/ddtrace/contrib/que/patcher_spec.rb new file mode 100644 index 00000000000..dc5c4b3fb8a --- /dev/null +++ b/spec/ddtrace/contrib/que/patcher_spec.rb @@ -0,0 +1,17 @@ +require 'ddtrace/contrib/support/spec_helper' +require 'ddtrace' +require 'que' + +RSpec.describe Datadog::Contrib::Que::Patcher do + describe '.patch' do + subject!(:patch) { described_class.patch } + + let(:middlewares) { ::Que.job_middleware.to_a } + + before do + described_class.patch + end + + it { expect(middlewares).to include(Datadog::Contrib::Que::Tracer) } + end +end diff --git a/spec/ddtrace/contrib/que/tracer_spec.rb b/spec/ddtrace/contrib/que/tracer_spec.rb new file mode 100644 index 00000000000..1b4a75129d3 --- /dev/null +++ b/spec/ddtrace/contrib/que/tracer_spec.rb @@ -0,0 +1,99 @@ +require 'ddtrace/contrib/support/spec_helper' +require 'ddtrace/contrib/analytics_examples' +require 'ddtrace' +require 'que' + +RSpec.describe Datadog::Contrib::Que::Tracer do + let(:job_args) do + { + field_one: 1, + queue: 'low', + priority: 10, + tags: { a: 1, b: 2 } + } + end + let(:job_class) do + stub_const('TestJobClass', Class.new(::Que::Job) do + def run(*args); end + end) + end + let(:error_job_class) do + stub_const('ErrorJobClass', Class.new(::Que::Job) do + def run(*args) + raise StandardError, 'with some error' + end + end) + end + + before do + Datadog.configure do |c| + c.use :que, configuration_options + end + + Que::Job.run_synchronously = true + end + + around do |example| + Datadog.registry[:que].reset_configuration! + example.run + Datadog.registry[:que].reset_configuration! + end + + describe '#call' do + context 'with default options' do + let(:configuration_options) { {} } + + it 'captures all generic span information' do + job_class.enqueue(job_args) + + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_QUEUE)).to eq(job_args[:queue]) + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_PRIORITY)).to eq(job_args[:priority]) + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_ERROR_COUNT)).to eq(0) + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_EXPIRED_AT)).to eq('') + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_FINISHED_AT)).to eq('') + end + + it 'does not capture info for disabled tags' do + job_class.enqueue(job_args) + + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_ARGS)).to eq(nil) + expect(span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_DATA)).to eq(nil) + end + + it 'continues to capture spans gracefully under unexpected conditions' do + expect { error_job_class.enqueue(job_args) }.to raise_error(StandardError) + expect(spans).not_to be_empty + expect(span.start_time).not_to be_nil + expect(span.end_time).not_to be_nil + expect(span.get_tag(Datadog::Ext::Errors::TYPE)).to eq('StandardError') + expect(span.get_tag(Datadog::Ext::Errors::STACK)).not_to be_nil + end + end + + context 'with tag_args enabled' do + let(:configuration_options) { { tag_args: true } } + + it 'captures span info for args tag' do + job_class.enqueue(job_args) + + actual_span_value = span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_ARGS) + expected_span_value = [{ field_one: 1 }].to_s + + expect(actual_span_value).to eq(expected_span_value) + end + end + + context 'with tag_data enabled' do + let(:configuration_options) { { tag_data: true } } + + it 'captures spans info for data tag' do + job_class.enqueue(job_args) + + actual_span_value = span.get_tag(Datadog::Contrib::Que::Ext::TAG_JOB_DATA) + expected_span_value = { tags: job_args[:tags] }.to_s + + expect(actual_span_value).to eq(expected_span_value) + end + end + end +end