From ed952022532faea05cf3d0abffa18a92902fd5f2 Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Tue, 6 Apr 2021 23:26:36 +0000 Subject: [PATCH 1/3] Do not query for unused data in BasicTypeMapping Usage of the range data was previously commented-out but was removed entirely in 365008ef71118da01dc9b3f422c9a81ae62602ad. --- lib/pg/basic_type_mapping.rb | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/lib/pg/basic_type_mapping.rb b/lib/pg/basic_type_mapping.rb index eb1c09eb2..67565c2e3 100644 --- a/lib/pg/basic_type_mapping.rb +++ b/lib/pg/basic_type_mapping.rb @@ -89,25 +89,12 @@ def coder_by_oid(oid) private - def supports_ranges?(connection) - connection.server_version >= 90200 - end - def build_coder_maps(connection) - if supports_ranges?(connection) - result = connection.exec <<-SQL - SELECT t.oid, t.typname, t.typelem, t.typdelim, ti.proname AS typinput, r.rngsubtype - FROM pg_type as t - JOIN pg_proc as ti ON ti.oid = t.typinput - LEFT JOIN pg_range as r ON t.oid = r.rngtypid - SQL - else - result = connection.exec <<-SQL - SELECT t.oid, t.typname, t.typelem, t.typdelim, ti.proname AS typinput - FROM pg_type as t - JOIN pg_proc as ti ON ti.oid = t.typinput - SQL - end + result = connection.exec <<-SQL + SELECT t.oid, t.typname, t.typelem, t.typdelim, ti.proname AS typinput + FROM pg_type as t + JOIN pg_proc as ti ON ti.oid = t.typinput + SQL [ [0, :encoder, PG::TextEncoder::Array], From 4f38d28f3ac6383a186ab92ac0d0c48efadf85ae Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Tue, 6 Apr 2021 23:14:43 +0000 Subject: [PATCH 2/3] Avoid materializing Result multiple times in type map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Testing methodology: - 189k rows returned from pg_type query in `build_coder_maps`. - Timing calling `PG::BasicTypeMapForQueries.new` only, figures are average of 16 iterations. runtime objects allocations Baseline : 2,027±350 ms 909 k 609 MiB Optimized: 874± 70 ms (-56%) 113 k (-88%) 113 MiB (-79%) Unfortunately performing the actual SQL query cannot easily be removed, so there is a fair bit of variance in the runtimes, however, results still show substantial improvement even when considering the worse case. If I had more time I would time the query and remove it, but that requires a bit more hacking than I would prefer, and, as said, the improvement stands either way. --- lib/pg/basic_type_mapping.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg/basic_type_mapping.rb b/lib/pg/basic_type_mapping.rb index 67565c2e3..ab0be2c6a 100644 --- a/lib/pg/basic_type_mapping.rb +++ b/lib/pg/basic_type_mapping.rb @@ -90,7 +90,7 @@ def coder_by_oid(oid) private def build_coder_maps(connection) - result = connection.exec <<-SQL + result = connection.exec(<<-SQL).to_a SELECT t.oid, t.typname, t.typelem, t.typdelim, ti.proname AS typinput FROM pg_type as t JOIN pg_proc as ti ON ti.oid = t.typinput From 4ac7211a2aaa664311a90624886c555d9f2658f6 Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Wed, 7 Apr 2021 21:52:08 +0000 Subject: [PATCH 3/3] Permit building coder maps ahead-of-time of TypeMap init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building coder maps can be quite expensive, building them ahead-of-time allows them to be reused for multiple type map creations, e.g. if creating both BasicTypeMapForQueries and BasicTypeMapForResults the cost need only be paid once. Example usage: conn = PG.connect(…) coder_maps = BasicTypeRegistry.build_coder_maps(conn) tmq = BasicTypeMapForQueries.new(conn, coder_maps: coder_maps) tmr = BasicTypeMapForResults.new(conn, coder_maps: coder_maps) Testing methodology: - 189k rows returned from pg_type query in `build_coder_maps`. - Cached and injected results from query to remove query cost and jitter from baseline. - Timing calling `PG::BasicTypeMapForQueries.new` only, figures are average of 32 iterations. - “Cached” calls `build_coder_maps` ahead-of-time, outside the timing loop, and passes-in. runtime Baseline: 374±16 ms Cached : <1± 0 ms (-99%) This confirms that the majority of the cost in creating a new type map is, when ignoring SQL query and results materialization, enclosed within `build_coder_maps`. As such, for *n* type maps, this reduces the (sometimes non-small) time complexity effectively from O(n) → O(1). --- lib/pg/basic_type_mapping.rb | 13 +++++-------- spec/pg/basic_type_mapping_spec.rb | 8 ++++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/pg/basic_type_mapping.rb b/lib/pg/basic_type_mapping.rb index ab0be2c6a..951870442 100644 --- a/lib/pg/basic_type_mapping.rb +++ b/lib/pg/basic_type_mapping.rb @@ -87,8 +87,6 @@ def coder_by_oid(oid) end end - private - def build_coder_maps(connection) result = connection.exec(<<-SQL).to_a SELECT t.oid, t.typname, t.typelem, t.typdelim, ti.proname AS typinput @@ -107,6 +105,7 @@ def build_coder_maps(connection) h end end + module_function :build_coder_maps ValidFormats = { 0 => true, 1 => true } ValidDirections = { :encoder => true, :decoder => true } @@ -122,8 +121,6 @@ def check_format_and_direction(format, direction) # objects as values. CODERS_BY_NAME = [] - public - # Register an encoder or decoder instance for casting a PostgreSQL type. # # Coder#name must correspond to the +typname+ column in the +pg_type+ table. @@ -290,8 +287,8 @@ def typecast_result_value(result, _tuple, field) end end - def initialize(connection) - @coder_maps = build_coder_maps(connection) + def initialize(connection, coder_maps: nil) + @coder_maps = coder_maps || build_coder_maps(connection) # Populate TypeMapByOid hash with decoders @coder_maps.flat_map{|f| f[:decoder].coders }.each do |coder| @@ -382,8 +379,8 @@ class BinaryData < String include PG::BasicTypeRegistry - def initialize(connection) - @coder_maps = build_coder_maps(connection) + def initialize(connection, coder_maps: nil) + @coder_maps = coder_maps || build_coder_maps(connection) @array_encoders_by_klass = array_encoders_by_klass @encode_array_as = :array init_encoders diff --git a/spec/pg/basic_type_mapping_spec.rb b/spec/pg/basic_type_mapping_spec.rb index bbcaff5bc..01c0c84f5 100644 --- a/spec/pg/basic_type_mapping_spec.rb +++ b/spec/pg/basic_type_mapping_spec.rb @@ -635,4 +635,12 @@ def to_s end end end + + describe PG::BasicTypeRegistry do + it "can build coder maps" do + expect do + described_class.build_coder_maps(@conn) + end.to_not raise_error + end + end end