Merge pull request #94 from co-cddo/gather_data_from_rapid

Gather data from rapid

Gemfile

@@ -58,6 +58,9 @@ gem "govuk_design_system_formbuilder"
 # Pagination
 gem "pagy", "~> 6.0"
 
+# HTTP client
+gem "faraday"
+
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem "debug", platforms: %i[mri mingw x64_mingw]

Gemfile.lock

@@ -92,6 +92,11 @@ GEM
       railties (>= 5.0.0)
     faker (3.2.0)
       i18n (>= 1.8.11, < 2)
+    faraday (2.10.0)
+      faraday-net_http (>= 2.0, < 3.2)
+      logger
+    faraday-net_http (3.1.0)
+      net-http
     globalid (1.2.1)
       activesupport (>= 6.1)
     govuk-components (5.1.0)
@@ -114,6 +119,7 @@ GEM
     jsbundling-rails (1.1.1)
       railties (>= 6.0.0)
     json (2.6.3)
+    logger (1.6.0)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -127,6 +133,8 @@ GEM
     mini_mime (1.1.5)
     minitest (5.22.2)
     msgpack (1.7.1)
+    net-http (0.4.1)
+      uri
     net-imap (0.4.10)
       date
       net-protocol
@@ -255,6 +263,7 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
+    uri (0.13.0)
     view_component (3.10.0)
       activesupport (>= 5.2.0, < 8.0)
       concurrent-ruby (~> 1.0)
@@ -283,6 +292,7 @@ DEPENDENCIES
   debug
   factory_bot_rails
   faker
+  faraday
   govuk-components
   govuk_design_system_formbuilder
   jsbundling-rails

app/controllers/agreements_controller.rb

@@ -9,7 +9,7 @@ def index
     agreements = agreements.where(id: [power.agreements.pluck(:id)]) if power
 
     agreements = agreements.where_first_letter(first_letter) if first_letter
-    agreements = agreements.where("fields ->> 'ISA_status' = :status", status: isa_status) if isa_status
+    agreements = agreements.where("fields ->> 'isa_status' = :status", status: isa_status) if isa_status
 
     # Using reorder because Agreement has a default scope that sets the order
     # Using unscoped would break the `control_person.agreements` association
@@ -27,9 +27,9 @@ def show
   def sort_by
     options = {
       name: :name,
-      id: Arel.sql("(fields ->> 'ID')::Integer"),
-      end_date: Arel.sql("(fields ->> 'End_date')::timestamptz"),
-      start_date: Arel.sql("(fields ->> 'Start_date')::timestamptz"),
+      id: Arel.sql("(fields ->> 'id')::Integer"),
+      end_date: Arel.sql("(fields ->> 'end_date')::timestamptz"),
+      start_date: Arel.sql("(fields ->> 'start_date')::timestamptz"),
     }
 
     return options[:id] if params[:sort_by].blank?

app/models/agreement.rb

@@ -1,5 +1,7 @@
-class Agreement < AirTable
+class Agreement < DataTable
   self.air_table_name = "Information Sharing Agreements"
+  self.rapid_table_name = :agreements
+  self.rapid_name_field = :agreement_name
 
   default_scope { order(Arel.sql("(fields ->> 'ID')::Integer")) }
 
@@ -13,10 +15,14 @@ class Agreement < AirTable
   has_many :processors, through: :agreement_processors
 
   def self.isa_statuses
-    pluck(Arel.sql("fields -> 'ISA_status'")).uniq
+    pluck(Arel.sql("fields -> 'isa_status'")).uniq
+  end
+
+  def self.find_by_id!(id)
+    find_by!("(fields ->> 'id')::Integer = ?", id.to_i)
   end
 
   def id_and_name
-    [fields["ID"], name].select(&:present?).join(" - ")
+    [fields["id"], name].select(&:present?).join(" - ")
   end
 end

app/models/agreement_control_person.rb

@@ -1,13 +1,31 @@
 class AgreementControlPerson < ApplicationRecord
+  RAPID_TABLE_NAME = :agreements_controllers
+
   belongs_to :agreement
   belongs_to :control_person
 
-  def self.populate
-    delete_all # Simplest way to ensure records deleted from Airtable do not persist in local database
-    Agreement.find_each do |agreement|
-      (agreement.fields["Controllers"] || []).each do |control_person_id|
-        control_person = ControlPerson.find_by(record_id: control_person_id)
-        find_or_create_by!(agreement:, control_person:) if control_person.present?
+  class << self
+    def populate
+      delete_all # Simplest way to ensure records deleted from Airtable do not persist in local database
+
+      air_table_data_source? ? populate_from_airtable : populate_from_rapid
+    end
+
+    def populate_from_airtable
+      Agreement.find_each do |agreement|
+        (agreement.fields["controllers"] || []).each do |control_person_id|
+          control_person = ControlPerson.find_by(record_id: control_person_id)
+          find_or_create_by!(agreement:, control_person:) if control_person.present?
+        end
+      end
+    end
+
+    def populate_from_rapid
+      RapidApi.output_for(RAPID_TABLE_NAME).each_value do |record|
+        agreement = Agreement.find_by_id!(record[:id])
+        control_person = ControlPerson.find_by!(name: record[:controller_name])
+
+        find_or_create_by!(agreement:, control_person:)
       end
     end
   end

app/models/agreement_processor.rb

@@ -1,13 +1,31 @@
 class AgreementProcessor < ApplicationRecord
+  RAPID_TABLE_NAME = :agreements_processors
+
   belongs_to :agreement
   belongs_to :processor
 
-  def self.populate
-    delete_all # Simplest way to ensure records deleted from Airtable do not persist in local database
-    Agreement.find_each do |agreement|
-      (agreement.fields["Processors"] || []).each do |processor_id|
-        processor = Processor.find_by(record_id: processor_id)
-        find_or_create_by!(agreement:, processor:) if processor.present?
+  class << self
+    def populate
+      delete_all # Simplest way to ensure records deleted from Airtable do not persist in local database
+
+      air_table_data_source? ? populate_from_airtable : populate_from_rapid
+    end
+
+    def populate_from_airtable
+      Agreement.find_each do |agreement|
+        (agreement.fields["processors"] || []).each do |processor_id|
+          processor = Processor.find_by(record_id: processor_id)
+          find_or_create_by!(agreement:, processor:) if processor.present?
+        end
+      end
+    end
+
+    def populate_from_rapid
+      RapidApi.output_for(RAPID_TABLE_NAME).each_value do |record|
+        agreement = Agreement.find_by_id!(record[:id])
+        processor = Processor.find_by!(name: record[:processor_name])
+
+        find_or_create_by!(agreement:, processor:)
       end
     end
   end

app/models/application_record.rb

@@ -2,4 +2,8 @@
 
 class ApplicationRecord < ActiveRecord::Base
   primary_abstract_class
+
+  def self.air_table_data_source?
+    Rails.configuration.data_source == :airtable
+  end
 end

app/models/concerns/airtable_data_source.rb

@@ -0,0 +1,41 @@
+# Use in data tables where source of data is (can be) Airtable.
+# Add the class via extend. For example:
+#
+#     class FooBar < DataTable
+#       extend AirtableDataSource
+#
+
+module AirtableDataSource
+  def search_via_air_table(text)
+    query = { "filterByFormula" => %[SEARCH("#{text}",{Name})] }
+    data = AirTableApi.data_for(air_table_path, query:)
+    return none if data[:records].empty?
+
+    ids = data[:records].pluck(:id)
+    where(record_id: ids)
+  end
+
+  def data_from_air_table
+    data = {}
+    records = {}
+    # API returns 100 records at a time.
+    # If there are more records, API returns an offset key that needs to be
+    # passed into the next query
+    while data.empty? || data[:offset].present?
+      query = data[:offset].present? ? { offset: data[:offset] } : {}
+      data = AirTableApi.data_for(air_table_path, query:)
+      data[:records].each do |record|
+        next if record[:fields].empty?
+        next if is_draft?(record)
+
+        records[record[:id]] = record[:fields]
+      end
+    end
+    records
+  end
+
+  def air_table_path
+    table_id = AirTableTable.id_for_name(air_table_name)
+    "#{AirTableBase.base_id}/#{table_id}"
+  end
+end

app/models/concerns/rapid_data_source.rb

@@ -0,0 +1,10 @@
+module RapidDataSource
+  def data_from_rapid
+    data = RapidApi.output_for(rapid_table_name)
+
+    data.each_with_object({}) do |(_key, record), hash|
+      record[:name] = record[rapid_name_field]
+      hash[record.fetch(:id, record[:name])] = record # Identifiy via the id within the record if present, else use name
+    end
+  end
+end

app/models/control_person.rb

@@ -1,5 +1,7 @@
-class ControlPerson < AirTable
+class ControlPerson < DataTable
   self.air_table_name = "Controllers"
+  self.rapid_table_name = :controllers
+  self.rapid_name_field = :controller_name
 
   has_many :power_control_people, dependent: :delete_all
   has_many :powers, through: :power_control_people