From d770aa7df3fe43dfccb93c19d568e898ddf967c4 Mon Sep 17 00:00:00 2001 From: Erica Porter Date: Fri, 19 Apr 2024 12:22:43 +0100 Subject: [PATCH 1/3] Explicitly truncate updated_at and created_at --- .../services/postgres_checksum_calculator.rb | 2 +- .../analytics/services/entity_table_checks_spec.rb | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/dfe/analytics/services/postgres_checksum_calculator.rb b/lib/dfe/analytics/services/postgres_checksum_calculator.rb index 8f4c0b69..339aded8 100644 --- a/lib/dfe/analytics/services/postgres_checksum_calculator.rb +++ b/lib/dfe/analytics/services/postgres_checksum_calculator.rb @@ -55,7 +55,7 @@ def build_select_and_order_clause(order_column, table_name_sanitized) end select_clause = case order_column when 'UPDATED_AT', 'CREATED_AT' - "#{table_name_sanitized}.#{order_column.downcase} AS \"#{order_alias}\"" + "DATE_TRUNC('milliseconds', #{table_name_sanitized}.#{order_column.downcase}) AS \"#{order_alias}\"" else "#{table_name_sanitized}.id::TEXT AS \"#{order_alias}\"" end diff --git a/spec/dfe/analytics/services/entity_table_checks_spec.rb b/spec/dfe/analytics/services/entity_table_checks_spec.rb index 7c8e1f3c..3b2be76a 100644 --- a/spec/dfe/analytics/services/entity_table_checks_spec.rb +++ b/spec/dfe/analytics/services/entity_table_checks_spec.rb @@ -215,5 +215,17 @@ })]) end end + + it 'orders records by updated_at truncated to milliseconds' do + time_base = Time.zone.now.beginning_of_minute + Candidate.create(email_address: 'first@example.com', updated_at: time_base + 0.001.seconds) + Candidate.create(email_address: 'second@example.com', updated_at: time_base + 0.005.seconds) + + described_class.call(entity_name: candidate_entity, entity_type: entity_type, entity_tag: nil) + + ordered_candidates = Candidate.order(:updated_at).pluck(:email_address) + + expect(ordered_candidates).to eq(['first@example.com', 'second@example.com']) + end end end From 83da45fdeb9db7ccc5b17a705199ace7067f52f5 Mon Sep 17 00:00:00 2001 From: Erica Porter Date: Mon, 22 Apr 2024 09:21:24 +0100 Subject: [PATCH 2/3] Add truncate to where clause --- .../analytics/services/generic_checksum_calculator.rb | 9 ++++++++- .../analytics/services/postgres_checksum_calculator.rb | 9 ++++++++- lib/dfe/analytics/shared/checksum_query_components.rb | 10 ---------- 3 files changed, 16 insertions(+), 12 deletions(-) delete mode 100644 lib/dfe/analytics/shared/checksum_query_components.rb diff --git a/lib/dfe/analytics/services/generic_checksum_calculator.rb b/lib/dfe/analytics/services/generic_checksum_calculator.rb index 02543808..bd6d38ee 100644 --- a/lib/dfe/analytics/services/generic_checksum_calculator.rb +++ b/lib/dfe/analytics/services/generic_checksum_calculator.rb @@ -8,7 +8,8 @@ module Services # and order column in a generic database class GenericChecksumCalculator include ServicePattern - include ChecksumQueryComponents + + WHERE_CLAUSE_ORDER_COLUMNS = %w[CREATED_AT UPDATED_AT].freeze def initialize(entity, order_column, checksum_calculated_at) @entity = entity @@ -58,6 +59,12 @@ def build_select_and_order_clause(order_column, table_name_sanitized) [select_clause, order_by_clause] end + + def build_where_clause(order_column, table_name_sanitized, checksum_calculated_at_sanitized) + return '' unless WHERE_CLAUSE_ORDER_COLUMNS.map(&:downcase).include?(order_column.downcase) + + "WHERE DATE_TRUNC('milliseconds', #{table_name_sanitized}.#{order_column.downcase}) < #{checksum_calculated_at_sanitized}" + end end end end diff --git a/lib/dfe/analytics/services/postgres_checksum_calculator.rb b/lib/dfe/analytics/services/postgres_checksum_calculator.rb index 339aded8..77333b74 100644 --- a/lib/dfe/analytics/services/postgres_checksum_calculator.rb +++ b/lib/dfe/analytics/services/postgres_checksum_calculator.rb @@ -8,7 +8,8 @@ module Services # and order column in a PostgreSQL database class PostgresChecksumCalculator include ServicePattern - include ChecksumQueryComponents + + WHERE_CLAUSE_ORDER_COLUMNS = %w[CREATED_AT UPDATED_AT].freeze def initialize(entity, order_column, checksum_calculated_at) @entity = entity @@ -61,6 +62,12 @@ def build_select_and_order_clause(order_column, table_name_sanitized) end [select_clause, order_alias] end + + def build_where_clause(order_column, table_name_sanitized, checksum_calculated_at_sanitized) + return '' unless WHERE_CLAUSE_ORDER_COLUMNS.map(&:downcase).include?(order_column.downcase) + + "WHERE #{table_name_sanitized}.#{order_column.downcase} < #{checksum_calculated_at_sanitized}" + end end end end diff --git a/lib/dfe/analytics/shared/checksum_query_components.rb b/lib/dfe/analytics/shared/checksum_query_components.rb deleted file mode 100644 index 2dfd2818..00000000 --- a/lib/dfe/analytics/shared/checksum_query_components.rb +++ /dev/null @@ -1,10 +0,0 @@ -# components used by checksum calculator -module ChecksumQueryComponents - WHERE_CLAUSE_ORDER_COLUMNS = %w[CREATED_AT UPDATED_AT].freeze - - def build_where_clause(order_column, table_name_sanitized, checksum_calculated_at_sanitized) - return '' unless WHERE_CLAUSE_ORDER_COLUMNS.map(&:downcase).include?(order_column.downcase) - - "WHERE #{table_name_sanitized}.#{order_column.downcase} < #{checksum_calculated_at_sanitized}" - end -end From d66cd0de155ff9e39f355aed249fcdebeadf68ba Mon Sep 17 00:00:00 2001 From: Erica Porter Date: Mon, 22 Apr 2024 10:48:51 +0100 Subject: [PATCH 3/3] Add truncate to checksum_calculated_at --- lib/dfe/analytics/services/generic_checksum_calculator.rb | 3 +-- lib/dfe/analytics/services/postgres_checksum_calculator.rb | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/dfe/analytics/services/generic_checksum_calculator.rb b/lib/dfe/analytics/services/generic_checksum_calculator.rb index bd6d38ee..069f55d3 100644 --- a/lib/dfe/analytics/services/generic_checksum_calculator.rb +++ b/lib/dfe/analytics/services/generic_checksum_calculator.rb @@ -1,5 +1,4 @@ require_relative '../shared/service_pattern' -require_relative '../shared/checksum_query_components' module DfE module Analytics @@ -63,7 +62,7 @@ def build_select_and_order_clause(order_column, table_name_sanitized) def build_where_clause(order_column, table_name_sanitized, checksum_calculated_at_sanitized) return '' unless WHERE_CLAUSE_ORDER_COLUMNS.map(&:downcase).include?(order_column.downcase) - "WHERE DATE_TRUNC('milliseconds', #{table_name_sanitized}.#{order_column.downcase}) < #{checksum_calculated_at_sanitized}" + "WHERE #{table_name_sanitized}.#{order_column.downcase} < #{checksum_calculated_at_sanitized}" end end end diff --git a/lib/dfe/analytics/services/postgres_checksum_calculator.rb b/lib/dfe/analytics/services/postgres_checksum_calculator.rb index 77333b74..e09caf6f 100644 --- a/lib/dfe/analytics/services/postgres_checksum_calculator.rb +++ b/lib/dfe/analytics/services/postgres_checksum_calculator.rb @@ -1,5 +1,4 @@ require_relative '../shared/service_pattern' -require_relative '../shared/checksum_query_components' module DfE module Analytics @@ -66,7 +65,7 @@ def build_select_and_order_clause(order_column, table_name_sanitized) def build_where_clause(order_column, table_name_sanitized, checksum_calculated_at_sanitized) return '' unless WHERE_CLAUSE_ORDER_COLUMNS.map(&:downcase).include?(order_column.downcase) - "WHERE #{table_name_sanitized}.#{order_column.downcase} < #{checksum_calculated_at_sanitized}" + "WHERE DATE_TRUNC('milliseconds', #{table_name_sanitized}.#{order_column.downcase}) < DATE_TRUNC('milliseconds', '#{checksum_calculated_at_sanitized}')" end end end