Add better error logging when BigQuery API call errors (#78)

DFE-Digital · Jun 28, 2023 · 6433de9 · 6433de9
1 parent 4d97047
commit 6433de9
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 11 deletions.
diff --git a/lib/dfe/analytics.rb b/lib/dfe/analytics.rb
@@ -70,10 +70,10 @@ def self.configure
       yield(config)
 
       config.enable_analytics                 ||= proc { true }
-      config.bigquery_table_name              ||= ENV['BIGQUERY_TABLE_NAME']
-      config.bigquery_project_id              ||= ENV['BIGQUERY_PROJECT_ID']
-      config.bigquery_dataset                 ||= ENV['BIGQUERY_DATASET']
-      config.bigquery_api_json_key            ||= ENV['BIGQUERY_API_JSON_KEY']
+      config.bigquery_table_name              ||= ENV.fetch('BIGQUERY_TABLE_NAME', nil)
+      config.bigquery_project_id              ||= ENV.fetch('BIGQUERY_PROJECT_ID', nil)
+      config.bigquery_dataset                 ||= ENV.fetch('BIGQUERY_DATASET', nil)
+      config.bigquery_api_json_key            ||= ENV.fetch('BIGQUERY_API_JSON_KEY', nil)
       config.bigquery_retries                 ||= 3
       config.bigquery_timeout                 ||= 120
       config.environment                      ||= ENV.fetch('RAILS_ENV', 'development')

diff --git a/lib/dfe/analytics/send_events.rb b/lib/dfe/analytics/send_events.rb
@@ -31,22 +31,28 @@ def perform(events)
           response = DfE::Analytics.events_client.insert(events, ignore_unknown: true)
 
           unless response.success?
-            error_message = error_message_for(response.insert_errors)
+            event_count   = events.length
+            error_message = error_message_for(response, events)
 
             Rails.logger.error(error_message)
 
+            events.each.with_index(1) do |event, index|
+              Rails.logger.info("DfE::Analytics possible error processing event (#{index}/#{event_count}): #{event.inspect}")
+            end
+
             raise SendEventsError, error_message
           end
         end
       end
 
-      def error_message_for(insert_errors)
-        message = insert_errors
-          .flat_map(&:errors)
-          .map { |error| error.try(:message) || error['message'] }
+      def error_message_for(resp, events)
+        message =
+          resp
+          .error_rows
+          .map { |row| "row: #{row} errors: #{resp.errors_for(row)} index: #{resp.index_for(row)} event: #{events[resp.index_for(row)].inspect}" }
           .compact.join("\n")
 
-        "Could not insert all events:\n#{message}"
+        "Could not insert #{resp.error_count} event(s):\n#{message}"
       end
     end
 

diff --git a/spec/dfe/analytics/send_events_spec.rb b/spec/dfe/analytics/send_events_spec.rb
@@ -60,7 +60,7 @@
       end
 
       it 'logs the error message' do
-        expect(Rails.logger).to receive(:error).with(/Could not insert all events:/)
+        expect(Rails.logger).to receive(:error).with(/Could not insert 1 event\(s\):/)
 
         perform
       rescue DfE::Analytics::SendEventsError