From dad72ce02ebdf09ad172777d2985de14001ad35d Mon Sep 17 00:00:00 2001 From: asatwal Date: Tue, 27 Jun 2023 20:48:41 +0100 Subject: [PATCH] Add better error logging when BigQuery API call errors --- lib/dfe/analytics.rb | 8 ++++---- lib/dfe/analytics/send_events.rb | 18 ++++++++++++------ spec/dfe/analytics/send_events_spec.rb | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/dfe/analytics.rb b/lib/dfe/analytics.rb index 740c5138..827a831e 100644 --- a/lib/dfe/analytics.rb +++ b/lib/dfe/analytics.rb @@ -70,10 +70,10 @@ def self.configure yield(config) config.enable_analytics ||= proc { true } - config.bigquery_table_name ||= ENV['BIGQUERY_TABLE_NAME'] - config.bigquery_project_id ||= ENV['BIGQUERY_PROJECT_ID'] - config.bigquery_dataset ||= ENV['BIGQUERY_DATASET'] - config.bigquery_api_json_key ||= ENV['BIGQUERY_API_JSON_KEY'] + config.bigquery_table_name ||= ENV.fetch('BIGQUERY_TABLE_NAME', nil) + config.bigquery_project_id ||= ENV.fetch('BIGQUERY_PROJECT_ID', nil) + config.bigquery_dataset ||= ENV.fetch('BIGQUERY_DATASET', nil) + config.bigquery_api_json_key ||= ENV.fetch('BIGQUERY_API_JSON_KEY', nil) config.bigquery_retries ||= 3 config.bigquery_timeout ||= 120 config.environment ||= ENV.fetch('RAILS_ENV', 'development') diff --git a/lib/dfe/analytics/send_events.rb b/lib/dfe/analytics/send_events.rb index 72730304..5d32531c 100644 --- a/lib/dfe/analytics/send_events.rb +++ b/lib/dfe/analytics/send_events.rb @@ -31,22 +31,28 @@ def perform(events) response = DfE::Analytics.events_client.insert(events, ignore_unknown: true) unless response.success? - error_message = error_message_for(response.insert_errors) + event_count = events.length + error_message = error_message_for(response, events) Rails.logger.error(error_message) + events.each.with_index(1) do |event, index| + Rails.logger.info("DfE::Analytics possible error processing event (#{index}/#{event_count}): #{event.inspect}") + end + raise SendEventsError, error_message end end end - def error_message_for(insert_errors) - message = insert_errors - .flat_map(&:errors) - .map { |error| error.try(:message) || error['message'] } + def error_message_for(resp, events) + message = + resp + .error_rows + .map { |row| "row: #{row} errors: #{resp.errors_for(row)} index: #{resp.index_for(row)} event: #{events[resp.index_for(row)].inspect}" } .compact.join("\n") - "Could not insert all events:\n#{message}" + "Could not insert #{resp.error_count} event(s):\n#{message}" end end diff --git a/spec/dfe/analytics/send_events_spec.rb b/spec/dfe/analytics/send_events_spec.rb index 43fd74bc..5234e7fb 100644 --- a/spec/dfe/analytics/send_events_spec.rb +++ b/spec/dfe/analytics/send_events_spec.rb @@ -60,7 +60,7 @@ end it 'logs the error message' do - expect(Rails.logger).to receive(:error).with(/Could not insert all events:/) + expect(Rails.logger).to receive(:error).with(/Could not insert 1 event\(s\):/) perform rescue DfE::Analytics::SendEventsError