diff --git a/ldclient-rb.gemspec b/ldclient-rb.gemspec index a7b64dc7..810987a4 100644 --- a/ldclient-rb.gemspec +++ b/ldclient-rb.gemspec @@ -34,8 +34,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency "listen", "~> 3.0" # see file_data_source.rb spec.add_runtime_dependency "json", [">= 1.8", "< 3"] - spec.add_runtime_dependency "faraday", [">= 0.9", "< 2"] - spec.add_runtime_dependency "faraday-http-cache", [">= 1.3.0", "< 3"] spec.add_runtime_dependency "semantic", "~> 1.6" spec.add_runtime_dependency "net-http-persistent", [">= 2.9", "< 4.0"] spec.add_runtime_dependency "concurrent-ruby", "~> 1.0" diff --git a/lib/ldclient-rb/cache_store.rb b/lib/ldclient-rb/cache_store.rb index 8451bb5f..b91b363d 100644 --- a/lib/ldclient-rb/cache_store.rb +++ b/lib/ldclient-rb/cache_store.rb @@ -2,11 +2,9 @@ module LaunchDarkly # - # A thread-safe in-memory store suitable for use with the Faraday caching HTTP client. Uses the - # concurrent-ruby gem's Map as the underlying cache. + # A thread-safe in-memory store that uses the same semantics that Faraday would expect, although we + # no longer use Faraday. This is used by Requestor, when we are not in a Rails environment. # - # @see https://github.com/plataformatec/faraday-http-cache - # @see https://github.com/ruby-concurrency # @private # class ThreadSafeMemoryStore diff --git a/lib/ldclient-rb/config.rb b/lib/ldclient-rb/config.rb index 34f4f67b..c7c42e56 100644 --- a/lib/ldclient-rb/config.rb +++ b/lib/ldclient-rb/config.rb @@ -53,7 +53,6 @@ def initialize(opts = {}) @use_ldd = opts.has_key?(:use_ldd) ? opts[:use_ldd] : Config.default_use_ldd @offline = opts.has_key?(:offline) ? opts[:offline] : Config.default_offline @poll_interval = opts.has_key?(:poll_interval) && opts[:poll_interval] > Config.default_poll_interval ? opts[:poll_interval] : Config.default_poll_interval - @proxy = opts[:proxy] || Config.default_proxy @all_attributes_private = opts[:all_attributes_private] || false @private_attribute_names = opts[:private_attribute_names] || [] @send_events = opts.has_key?(:send_events) ? opts[:send_events] : Config.default_send_events @@ -153,9 +152,10 @@ def offline? attr_reader :capacity # - # A store for HTTP caching. This must support the semantics used by the - # [`faraday-http-cache`](https://github.com/plataformatec/faraday-http-cache) gem. Defaults - # to the Rails cache in a Rails environment, or a thread-safe in-memory store otherwise. + # A store for HTTP caching (used only in polling mode). This must support the semantics used by + # the [`faraday-http-cache`](https://github.com/plataformatec/faraday-http-cache) gem, although + # the SDK no longer uses Faraday. Defaults to the Rails cache in a Rails environment, or a + # thread-safe in-memory store otherwise. # @return [Object] # attr_reader :cache_store @@ -184,12 +184,6 @@ def offline? # attr_reader :feature_store - # - # The proxy configuration string. - # @return [String] - # - attr_reader :proxy - # # True if all user attributes (other than the key) should be considered private. This means # that the attribute values will not be sent to LaunchDarkly in analytics events and will not @@ -336,14 +330,6 @@ def self.default_connect_timeout 2 end - # - # The default value for {#proxy}. - # @return [String] nil - # - def self.default_proxy - nil - end - # # The default value for {#logger}. # @return [Logger] the Rails logger if in Rails, or a default Logger at WARN level otherwise diff --git a/lib/ldclient-rb/events.rb b/lib/ldclient-rb/events.rb index cbae5ac5..02885904 100644 --- a/lib/ldclient-rb/events.rb +++ b/lib/ldclient-rb/events.rb @@ -1,9 +1,9 @@ require "concurrent" require "concurrent/atomics" require "concurrent/executors" +require "net/http/persistent" require "thread" require "time" -require "faraday" module LaunchDarkly MAX_FLUSH_WORKERS = 5 @@ -115,7 +115,12 @@ class EventDispatcher def initialize(queue, sdk_key, config, client) @sdk_key = sdk_key @config = config - @client = client ? client : Faraday.new + + @client = client ? client : Net::HTTP::Persistent.new do |c| + c.open_timeout = @config.connect_timeout + c.read_timeout = @config.read_timeout + end + @user_keys = SimpleLRUCacheSet.new(config.user_keys_capacity) @formatter = EventOutputFormatter.new(config) @disabled = Concurrent::AtomicBoolean.new(false) @@ -162,7 +167,7 @@ def main_loop(queue, buffer, flush_workers) def do_shutdown(flush_workers) flush_workers.shutdown flush_workers.wait_for_termination - # There seems to be no such thing as "close" in Faraday: https://github.com/lostisland/faraday/issues/241 + @client.shutdown end def synchronize_for_testing(flush_workers) @@ -246,16 +251,17 @@ def trigger_flush(buffer, flush_workers) end def handle_response(res) - if res.status >= 400 - message = Util.http_error_message(res.status, "event delivery", "some events were dropped") + status = res.code.to_i + if status >= 400 + message = Util.http_error_message(status, "event delivery", "some events were dropped") @config.logger.error { "[LDClient] #{message}" } - if !Util.http_error_recoverable?(res.status) + if !Util.http_error_recoverable?(status) @disabled.value = true end else - if !res.headers.nil? && res.headers.has_key?("Date") + if !res["date"].nil? begin - res_time = (Time.httpdate(res.headers["Date"]).to_f * 1000).to_i + res_time = (Time.httpdate(res["date"]).to_f * 1000).to_i @last_known_past_time.value = res_time rescue ArgumentError end @@ -317,21 +323,21 @@ def run(sdk_key, config, client, payload, formatter) end begin config.logger.debug { "[LDClient] sending #{events_out.length} events: #{body}" } - res = client.post (config.events_uri + "/bulk") do |req| - req.headers["Authorization"] = sdk_key - req.headers["User-Agent"] = "RubyClient/" + LaunchDarkly::VERSION - req.headers["Content-Type"] = "application/json" - req.headers["X-LaunchDarkly-Event-Schema"] = CURRENT_SCHEMA_VERSION.to_s - req.body = body - req.options.timeout = config.read_timeout - req.options.open_timeout = config.connect_timeout - end + uri = URI(config.events_uri + "/bulk") + req = Net::HTTP::Post.new(uri) + req.content_type = "application/json" + req.body = body + req["Authorization"] = sdk_key + req["User-Agent"] = "RubyClient/" + LaunchDarkly::VERSION + req["X-LaunchDarkly-Event-Schema"] = CURRENT_SCHEMA_VERSION.to_s + res = client.request(uri, req) rescue StandardError => exn config.logger.warn { "[LDClient] Error flushing events: #{exn.inspect}." } next end - if res.status < 200 || res.status >= 300 - if Util.http_error_recoverable?(res.status) + status = res.code.to_i + if status < 200 || status >= 300 + if Util.http_error_recoverable?(status) next end end diff --git a/lib/ldclient-rb/polling.rb b/lib/ldclient-rb/polling.rb index 4c6769f3..17ff7c12 100644 --- a/lib/ldclient-rb/polling.rb +++ b/lib/ldclient-rb/polling.rb @@ -63,8 +63,7 @@ def create_worker stop end rescue StandardError => exn - @config.logger.error { "[LDClient] Exception while polling: #{exn.inspect}" } - # TODO: log_exception(__method__.to_s, exn) + Util.log_exception(@config.logger, "Exception while polling", exn) end delta = @config.poll_interval - (Time.now - started_at) if delta > 0 diff --git a/lib/ldclient-rb/requestor.rb b/lib/ldclient-rb/requestor.rb index 8922e82c..5f48d7ff 100644 --- a/lib/ldclient-rb/requestor.rb +++ b/lib/ldclient-rb/requestor.rb @@ -1,12 +1,13 @@ +require "concurrent/atomics" require "json" require "net/http/persistent" -require "faraday/http_cache" module LaunchDarkly # @private class UnexpectedResponseError < StandardError def initialize(status) @status = status + super("HTTP error #{status}") end def status @@ -16,14 +17,15 @@ def status # @private class Requestor + CacheEntry = Struct.new(:etag, :body) + def initialize(sdk_key, config) @sdk_key = sdk_key @config = config - @client = Faraday.new do |builder| - builder.use :http_cache, store: @config.cache_store, serializer: Marshal - - builder.adapter :net_http_persistent - end + @client = Net::HTTP::Persistent.new + @client.open_timeout = @config.connect_timeout + @client.read_timeout = @config.read_timeout + @cache = @config.cache_store end def request_flag(key) @@ -38,27 +40,59 @@ def request_all_data() make_request("/sdk/latest-all") end + def stop + @client.shutdown + end + + private + def make_request(path) - uri = @config.base_uri + path - res = @client.get (uri) do |req| - req.headers["Authorization"] = @sdk_key - req.headers["User-Agent"] = "RubyClient/" + LaunchDarkly::VERSION - req.options.timeout = @config.read_timeout - req.options.open_timeout = @config.connect_timeout - if @config.proxy - req.options.proxy = Faraday::ProxyOptions.from @config.proxy - end + uri = URI(@config.base_uri + path) + req = Net::HTTP::Get.new(uri) + req["Authorization"] = @sdk_key + req["User-Agent"] = "RubyClient/" + LaunchDarkly::VERSION + cached = @cache.read(uri) + if !cached.nil? + req["If-None-Match"] = cached.etag end + res = @client.request(uri, req) + status = res.code.to_i + @config.logger.debug { "[LDClient] Got response from uri: #{uri}\n\tstatus code: #{status}\n\theaders: #{res.to_hash}\n\tbody: #{res.body}" } - @config.logger.debug { "[LDClient] Got response from uri: #{uri}\n\tstatus code: #{res.status}\n\theaders: #{res.headers}\n\tbody: #{res.body}" } - - if res.status < 200 || res.status >= 300 - raise UnexpectedResponseError.new(res.status) + if status == 304 && !cached.nil? + body = cached.body + else + @cache.delete(uri) + if status < 200 || status >= 300 + raise UnexpectedResponseError.new(status) + end + body = fix_encoding(res.body, res["content-type"]) + etag = res["etag"] + @cache.write(uri, CacheEntry.new(etag, body)) if !etag.nil? end + JSON.parse(body, symbolize_names: true) + end - JSON.parse(res.body, symbolize_names: true) + def fix_encoding(body, content_type) + return body if content_type.nil? + media_type, charset = parse_content_type(content_type) + return body if charset.nil? + body.force_encoding(Encoding::find(charset)).encode(Encoding::UTF_8) end - private :make_request + def parse_content_type(value) + return [nil, nil] if value.nil? || value == '' + parts = value.split(/; */) + return [value, nil] if parts.count < 2 + charset = nil + parts.each do |part| + fields = part.split('=') + if fields.count >= 2 && fields[0] == 'charset' + charset = fields[1] + break + end + end + return [parts[0], charset] + end end end diff --git a/lib/ldclient-rb/stream.rb b/lib/ldclient-rb/stream.rb index 094a37b2..ddb7f669 100644 --- a/lib/ldclient-rb/stream.rb +++ b/lib/ldclient-rb/stream.rb @@ -50,7 +50,6 @@ def start } opts = { headers: headers, - proxy: @config.proxy, read_timeout: READ_TIMEOUT_SECONDS, logger: @config.logger } diff --git a/spec/events_spec.rb b/spec/events_spec.rb index 56bd14a2..86cc67b6 100644 --- a/spec/events_spec.rb +++ b/spec/events_spec.rb @@ -1,5 +1,5 @@ +require "http_util" require "spec_helper" -require "faraday" require "time" describe LaunchDarkly::EventProcessor do @@ -348,7 +348,7 @@ @ep.flush @ep.wait_until_inactive - expect(hc.get_request.headers["Authorization"]).to eq "sdk_key" + expect(hc.get_request["authorization"]).to eq "sdk_key" end def verify_unrecoverable_http_error(status) @@ -414,7 +414,7 @@ def verify_recoverable_http_error(status) e = { kind: "identify", user: user } @ep.add_event(e) - hc.set_exception(Faraday::Error::ConnectionFailed.new("fail")) + hc.set_exception(IOError.new("deliberate error")) @ep.flush @ep.wait_until_inactive @@ -423,6 +423,46 @@ def verify_recoverable_http_error(status) expect(hc.get_request).to be_nil # no 3rd request end + it "makes actual HTTP request with correct headers" do + e = { kind: "identify", key: user[:key], user: user } + with_server do |server| + server.setup_ok_response("/bulk", "") + + @ep = subject.new("sdk_key", LaunchDarkly::Config.new(events_uri: server.base_uri.to_s)) + @ep.add_event(e) + @ep.flush + + req = server.await_request + expect(req.header).to include({ + "authorization" => [ "sdk_key" ], + "content-type" => [ "application/json" ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ], + "x-launchdarkly-event-schema" => [ "3" ] + }) + end + end + + it "can use a proxy server" do + e = { kind: "identify", key: user[:key], user: user } + with_server do |server| + server.setup_ok_response("/bulk", "") + + with_server(StubProxyServer.new) do |proxy| + begin + ENV["http_proxy"] = proxy.base_uri.to_s + @ep = subject.new("sdk_key", LaunchDarkly::Config.new(events_uri: server.base_uri.to_s)) + @ep.add_event(e) + @ep.flush + + req = server.await_request + expect(req["content-type"]).to eq("application/json") + ensure + ENV["http_proxy"] = nil + end + end + end + end + def index_event(e, user) { kind: "index", @@ -496,38 +536,35 @@ def reset @status = 200 end - def post(uri) - req = Faraday::Request.create("POST") - req.headers = {} - req.options = Faraday::RequestOptions.new - yield req + def request(uri, req) @requests.push(req) if @exception raise @exception else - resp = Faraday::Response.new headers = {} if @server_time headers["Date"] = @server_time.httpdate end - resp.finish({ - status: @status ? @status : 200, - response_headers: headers - }) - resp + FakeResponse.new(@status ? @status : 200, headers) end end def get_request @requests.shift end + + def shutdown + end end class FakeResponse - def initialize(status) - @status = status - end + include Net::HTTPHeader - attr_reader :status + attr_reader :code + + def initialize(status, headers) + @code = status.to_s + initialize_http_header(headers) + end end end diff --git a/spec/http_util.rb b/spec/http_util.rb index 764f8e48..e43e2ded 100644 --- a/spec/http_util.rb +++ b/spec/http_util.rb @@ -23,6 +23,7 @@ def initialize retry end @requests = [] + @requests_queue = Queue.new end def self.next_port @@ -62,6 +63,11 @@ def setup_ok_response(uri_path, body, content_type=nil, headers={}) def record_request(req, res) @requests.push(req) + @requests_queue << req + end + + def await_request + @requests_queue.pop end end diff --git a/spec/requestor_spec.rb b/spec/requestor_spec.rb index 7f2b8ad7..502f6d86 100644 --- a/spec/requestor_spec.rb +++ b/spec/requestor_spec.rb @@ -1,57 +1,212 @@ require "http_util" require "spec_helper" +$sdk_key = "secret" + describe LaunchDarkly::Requestor do - describe ".request_all_flags" do - describe "with a proxy" do - it "converts the proxy option" do - content = '{"flags": {"flagkey": {"key": "flagkey"}}}' - with_server do |server| - server.setup_ok_response("/sdk/latest-all", content, "application/json", { "etag" => "x" }) - with_server(StubProxyServer.new) do |proxy| - config = LaunchDarkly::Config.new(base_uri: server.base_uri.to_s, proxy: proxy.base_uri.to_s) - r = LaunchDarkly::Requestor.new("sdk-key", config) - result = r.request_all_data - expect(result).to eq(JSON.parse(content, symbolize_names: true)) - end - end - end - end - describe "without a proxy" do - it "sends headers" do - content = '{"flags": {}}' - sdk_key = 'sdk-key' - with_server do |server| - server.setup_ok_response("/sdk/latest-all", content, "application/json", { "etag" => "x" }) - r = LaunchDarkly::Requestor.new(sdk_key, LaunchDarkly::Config.new({ base_uri: server.base_uri.to_s })) - r.request_all_data - expect(server.requests.length).to eq 1 - req = server.requests[0] - expect(req.header['authorization']).to eq [sdk_key] - expect(req.header['user-agent']).to eq ["RubyClient/" + LaunchDarkly::VERSION] - end - end - - it "receives data" do - content = '{"flags": {"flagkey": {"key": "flagkey"}}}' - with_server do |server| - server.setup_ok_response("/sdk/latest-all", content, "application/json", { "etag" => "x" }) - r = LaunchDarkly::Requestor.new("sdk-key", LaunchDarkly::Config.new({ base_uri: server.base_uri.to_s })) - result = r.request_all_data - expect(result).to eq(JSON.parse(content, symbolize_names: true)) - end - end - - it "handles Unicode content" do - content = '{"flags": {"flagkey": {"key": "flagkey", "variations": ["blue", "grėeń"]}}}' - with_server do |server| - server.setup_ok_response("/sdk/latest-all", content, "application/json", { "etag" => "x" }) - # Note that the ETag header here is important because without it, the HTTP cache will not be used, - # and the cache is what required a fix to handle Unicode properly. See: - # https://github.com/launchdarkly/ruby-client/issues/90 - r = LaunchDarkly::Requestor.new("sdk-key", LaunchDarkly::Config.new({ base_uri: server.base_uri.to_s })) - result = r.request_all_data - expect(result).to eq(JSON.parse(content, symbolize_names: true)) + def with_requestor(base_uri) + r = LaunchDarkly::Requestor.new($sdk_key, LaunchDarkly::Config.new(base_uri: base_uri)) + yield r + r.stop + end + + describe "request_all_flags" do + it "uses expected URI and headers" do + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_ok_response("/", "{}") + requestor.request_all_data() + expect(server.requests.count).to eq 1 + expect(server.requests[0].unparsed_uri).to eq "/sdk/latest-all" + expect(server.requests[0].header).to include({ + "authorization" => [ $sdk_key ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ] + }) + end + end + end + + it "parses response" do + expected_data = { flags: { x: { key: "x" } } } + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_ok_response("/", expected_data.to_json) + data = requestor.request_all_data() + expect(data).to eq expected_data + end + end + end + + it "sends etag from previous response" do + etag = "xyz" + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_response("/") do |req, res| + res.status = 200 + res.body = "{}" + res["ETag"] = etag + end + requestor.request_all_data() + expect(server.requests.count).to eq 1 + + requestor.request_all_data() + expect(server.requests.count).to eq 2 + expect(server.requests[1].header).to include({ "if-none-match" => [ etag ] }) + end + end + end + + it "can reuse cached data" do + etag = "xyz" + expected_data = { flags: { x: { key: "x" } } } + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_response("/") do |req, res| + res.status = 200 + res.body = expected_data.to_json + res["ETag"] = etag + end + requestor.request_all_data() + expect(server.requests.count).to eq 1 + + server.setup_response("/") do |req, res| + res.status = 304 + end + data = requestor.request_all_data() + expect(server.requests.count).to eq 2 + expect(server.requests[1].header).to include({ "if-none-match" => [ etag ] }) + expect(data).to eq expected_data + end + end + end + + it "replaces cached data with new data" do + etag1 = "abc" + etag2 = "xyz" + expected_data1 = { flags: { x: { key: "x" } } } + expected_data2 = { flags: { y: { key: "y" } } } + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_response("/") do |req, res| + res.status = 200 + res.body = expected_data1.to_json + res["ETag"] = etag1 + end + data = requestor.request_all_data() + expect(data).to eq expected_data1 + expect(server.requests.count).to eq 1 + + server.setup_response("/") do |req, res| + res.status = 304 + end + data = requestor.request_all_data() + expect(data).to eq expected_data1 + expect(server.requests.count).to eq 2 + expect(server.requests[1].header).to include({ "if-none-match" => [ etag1 ] }) + + server.setup_response("/") do |req, res| + res.status = 200 + res.body = expected_data2.to_json + res["ETag"] = etag2 + end + data = requestor.request_all_data() + expect(data).to eq expected_data2 + expect(server.requests.count).to eq 3 + expect(server.requests[2].header).to include({ "if-none-match" => [ etag1 ] }) + + server.setup_response("/") do |req, res| + res.status = 304 + end + data = requestor.request_all_data() + expect(data).to eq expected_data2 + expect(server.requests.count).to eq 4 + expect(server.requests[3].header).to include({ "if-none-match" => [ etag2 ] }) + end + end + end + + it "uses UTF-8 encoding by default" do + content = '{"flags": {"flagkey": {"key": "flagkey", "variations": ["blue", "grėeń"]}}}' + with_server do |server| + server.setup_ok_response("/sdk/latest-all", content, "application/json") + with_requestor(server.base_uri.to_s) do |requestor| + data = requestor.request_all_data + expect(data).to eq(JSON.parse(content, symbolize_names: true)) + end + end + end + + it "detects other encodings from Content-Type" do + content = '{"flags": {"flagkey": {"key": "flagkey", "variations": ["proszę", "dziękuję"]}}}' + with_server do |server| + server.setup_ok_response("/sdk/latest-all", content.encode(Encoding::ISO_8859_2), + "text/plain; charset=ISO-8859-2") + with_requestor(server.base_uri.to_s) do |requestor| + data = requestor.request_all_data + expect(data).to eq(JSON.parse(content, symbolize_names: true)) + end + end + end + + it "throws exception for error status" do + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_response("/") do |req, res| + res.status = 400 + end + expect { requestor.request_all_data() }.to raise_error(LaunchDarkly::UnexpectedResponseError) + end + end + end + + it "can use a proxy server" do + content = '{"flags": {"flagkey": {"key": "flagkey"}}}' + with_server do |server| + server.setup_ok_response("/sdk/latest-all", content, "application/json", { "etag" => "x" }) + with_server(StubProxyServer.new) do |proxy| + begin + ENV["http_proxy"] = proxy.base_uri.to_s + with_requestor(server.base_uri.to_s) do |requestor| + data = requestor.request_all_data + expect(data).to eq(JSON.parse(content, symbolize_names: true)) + end + ensure + ENV["http_proxy"] = nil + end + end + end + end + end + + describe "request_flag" do + it "uses expected URI and headers" do + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_ok_response("/", "{}") + requestor.request_flag("key") + expect(server.requests.count).to eq 1 + expect(server.requests[0].unparsed_uri).to eq "/sdk/latest-flags/key" + expect(server.requests[0].header).to include({ + "authorization" => [ $sdk_key ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ] + }) + end + end + end + end + + describe "request_segment" do + it "uses expected URI and headers" do + with_server do |server| + with_requestor(server.base_uri.to_s) do |requestor| + server.setup_ok_response("/", "{}") + requestor.request_segment("key") + expect(server.requests.count).to eq 1 + expect(server.requests[0].unparsed_uri).to eq "/sdk/latest-segments/key" + expect(server.requests[0].header).to include({ + "authorization" => [ $sdk_key ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ] + }) end end end