Refactor scorecard degree program fetch (#794)

* refactor scorecard degree program fetch

* lint

* Update schema.rb

* unit tests

.rubocop.yml

@@ -96,6 +96,7 @@ Metrics/MethodLength:
     - "app/controllers/v0/institutions_controller.rb"
     - "lib/roo_helper/loader.rb"
     - "lib/seed_utils.rb"
+    - "app/utilities/scorecard_api/service.rb"
 
 Metrics/ClassLength:
   Exclude:

app/controllers/dashboards_controller.rb

@@ -123,7 +123,7 @@ def fetch_api_data(api_upload)
       message = "#{klass.name}::POPULATE_SUCCESS_MESSAGE".safe_constantize
       return message if message.present?
 
-      "#{klass.name} finished fetching data from it's api"
+      "#{klass.name} finished fetching data from its api"
     end
   end
 end

app/models/scorecard.rb

@@ -156,7 +156,7 @@ class Scorecard < ImportableRecord
   API_SOURCE = 'https://collegescorecard.ed.gov/data/'
 
   def self.populate
-    results = ScorecardApi::Service.populate('scorecard')
+    results = ScorecardApi::Service.populate
     load(results) if results.any?
     results.any?
   end

app/models/scorecard_degree_program.rb

@@ -16,7 +16,7 @@ class ScorecardDegreeProgram < ImportableRecord
   API_SOURCE = 'https://collegescorecard.ed.gov/data/'
 
   def self.populate
-    results = ScorecardApi::Service.populate('scorecard degree program')
+    results = ScorecardDegreeProgramApi::Service.populate
     load(results) if results.any?
     results.any?
   end

app/utilities/scorecard_api/service.rb

@@ -8,63 +8,46 @@ class Service
     # https://github.com/RTICWDT/open-data-maker/blob/master/lib/data_magic/query_builder.rb#L15
     MAX_PAGE_SIZE = 100
 
-    API_MAPPINGS = {
-      id: :cross,
-      ope8_id: :ope,
-      ope6_id: :ope6,
-      'latest.aid.federal_loan_rate': :pctfloan,
-      'latest.aid.median_debt_suppressed.completers.overall': :avg_stu_loan_debt,
-      'latest.completion.rate_suppressed.four_year': :c150_4_pooled_supp,
-      'latest.completion.rate_suppressed.lt_four_year_150percent': :c150_l4_pooled_supp,
-      'latest.earnings.10_yrs_after_entry.median': :salary_all_students,
-      'latest.repayment.3_yr_repayment_suppressed.overall': :repayment_rate_all_students,
-      'latest.student.retention_rate.four_year.full_time': :retention_all_students_ba,
-      'latest.student.retention_rate.lt_four_year.full_time': :retention_all_students_otb,
-      'latest.student.size': :undergrad_enrollment,
-      'location.lat': :latitude,
-      'location.lon': :longitude,
-      'school.school_url': :insturl,
-      'school.degrees_awarded.predominant': :pred_degree_awarded,
-      'school.locale': :locale,
-      'school.minority_serving.historically_black': :hbcu,
-      'school.men_only': :menonly,
-      'school.women_only': :womenonly,
-      'school.religious_affiliation': :relaffil,
-      'school.under_investigation': :hcm2,
-      'school.alias': :alias
-    }.freeze
-
-    DEGREE_PROGRAMS_API_MAPPINGS = {
-      'latest.programs.cip_4_digit.unit_id': :unit_id,
-      'latest.programs.cip_4_digit.ope6_id': :ope6_id,
-      'latest.programs.cip_4_digit.school.type': :control,
-      'latest.programs.cip_4_digit.school.main_campus': :main,
-      'latest.programs.cip_4_digit.code': :cipcode,
-      'latest.programs.cip_4_digit.title': :cipdesc,
-      'latest.programs.cip_4_digit.credential.level': :credlev,
-      'latest.programs.cip_4_digit.credential.title': :creddesc
-    }.freeze
-
-    def self.populate(result_type)
-      degree_programs = result_type == 'scorecard degree program'
+    def self.api_mappings
+      { id: :cross,
+        ope8_id: :ope,
+        ope6_id: :ope6,
+        'latest.aid.federal_loan_rate': :pctfloan,
+        'latest.aid.median_debt_suppressed.completers.overall': :avg_stu_loan_debt,
+        'latest.completion.rate_suppressed.four_year': :c150_4_pooled_supp,
+        'latest.completion.rate_suppressed.lt_four_year_150percent': :c150_l4_pooled_supp,
+        'latest.earnings.10_yrs_after_entry.median': :salary_all_students,
+        'latest.repayment.3_yr_repayment_suppressed.overall': :repayment_rate_all_students,
+        'latest.student.retention_rate.four_year.full_time': :retention_all_students_ba,
+        'latest.student.retention_rate.lt_four_year.full_time': :retention_all_students_otb,
+        'latest.student.size': :undergrad_enrollment,
+        'location.lat': :latitude,
+        'location.lon': :longitude,
+        'school.school_url': :insturl,
+        'school.degrees_awarded.predominant': :pred_degree_awarded,
+        'school.locale': :locale,
+        'school.minority_serving.historically_black': :hbcu,
+        'school.men_only': :menonly,
+        'school.women_only': :womenonly,
+        'school.religious_affiliation': :relaffil,
+        'school.under_investigation': :hcm2,
+        'school.alias': :alias }.freeze
+    end
 
+    def self.populate
       results = []
-      response_body = schools_api_call(0, degree_programs) #  call for page 0 to get initial @total
+      response_body = schools_api_call(0) #  call for page 0 to get initial @total
       results.push(*response_body[:results])
       number_of_pages = (response_body[:metadata][:total] / MAX_PAGE_SIZE).to_f.ceil
 
-      (1..number_of_pages).each { |page_num| results.push(*schools_api_call(page_num, degree_programs)[:results]) }
+      (1..number_of_pages).each { |page_num| results.push(*schools_api_call(page_num)[:results]) }
 
-      if degree_programs
-        map_degree_program_results(results)
-      else
-        map_results(results)
-      end
+      map_results(results)
     end
 
-    def self.schools_api_call(page, degree_programs)
+    def self.schools_api_call(page)
       params = {
-        'fields': !degree_programs ? API_MAPPINGS.keys.join(',') : DEGREE_PROGRAMS_API_MAPPINGS.keys.join(','),
+        'fields': api_mappings.keys.join(','),
         'per_page': MAX_PAGE_SIZE.to_s,
         'page': page
       }
@@ -78,31 +61,10 @@ def self.client
     def self.map_results(results)
       results.map do |result|
         scorecard = Scorecard.new
-        result.each_pair { |key, value| scorecard[API_MAPPINGS[key]] = value }
+        result.each_pair { |key, value| scorecard[api_mappings[key]] = value }
         scorecard.derive_dependent_columns
         scorecard
       end
     end
-
-    def self.map_degree_program_results(results)
-      degree_program_results = []
-      results.each do |result|
-        next unless result.key?(:'latest.programs.cip_4_digit')
-
-        result[:'latest.programs.cip_4_digit'].each do |degree_program|
-          scorecard_degree_program = ScorecardDegreeProgram.new
-          scorecard_degree_program[:unitid] = degree_program[:unit_id]
-          scorecard_degree_program[:ope6_id] = degree_program[:ope6_id]
-          scorecard_degree_program[:control] = degree_program[:school][:type]
-          scorecard_degree_program[:main] = degree_program[:school][:main_campus]
-          scorecard_degree_program[:cip_code] = degree_program[:code]
-          scorecard_degree_program[:cip_desc] = degree_program[:title]
-          scorecard_degree_program[:cred_lev] = degree_program[:credential][:level]
-          scorecard_degree_program[:cred_desc] = degree_program[:credential][:title]
-          degree_program_results.push(scorecard_degree_program)
-        end
-      end
-      degree_program_results
-    end
   end
 end

app/utilities/scorecard_degree_program_api/service.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'scorecard_api/client'
+
+module ScorecardDegreeProgramApi
+  class Service < ScorecardApi::Service
+    def self.api_mappings
+      {
+        'latest.programs.cip_4_digit.unit_id': :unit_id,
+        'latest.programs.cip_4_digit.ope6_id': :ope6_id,
+        'latest.programs.cip_4_digit.school.type': :control,
+        'latest.programs.cip_4_digit.school.main_campus': :main,
+        'latest.programs.cip_4_digit.code': :cipcode,
+        'latest.programs.cip_4_digit.title': :cipdesc,
+        'latest.programs.cip_4_digit.credential.level': :credlev,
+        'latest.programs.cip_4_digit.credential.title': :creddesc
+      }.freeze
+    end
+
+    def self.map_results(results)
+      degree_program_results = []
+      results.map do |result|
+        next unless result.key?(:'latest.programs.cip_4_digit')
+
+        result[:'latest.programs.cip_4_digit'].each do |degree_program|
+          scorecard_degree_program = ScorecardDegreeProgram.new
+          scorecard_degree_program[:unitid] = degree_program[:unit_id]
+          scorecard_degree_program[:ope6_id] = degree_program[:ope6_id]
+          scorecard_degree_program[:control] = degree_program[:school][:type]
+          scorecard_degree_program[:main] = degree_program[:school][:main_campus]
+          scorecard_degree_program[:cip_code] = degree_program[:code]
+          scorecard_degree_program[:cip_desc] = degree_program[:title]
+          scorecard_degree_program[:cred_lev] = degree_program[:credential][:level]
+          scorecard_degree_program[:cred_desc] = degree_program[:credential][:title]
+          degree_program_results.push(scorecard_degree_program)
+        end
+      end
+      degree_program_results
+    end
+  end
+end

config/initializers/csv_types.rb

@@ -15,7 +15,7 @@
   { klass: Mou, required?: true },
   { klass: Outcome, required?: true },
   { klass: Scorecard, required?: true, has_api?: true },
-  { klass: ScorecardDegreeProgram, required?: true, has_api?: true },
+  { klass: ScorecardDegreeProgram, required?: false, has_api?: true },
   { klass: Sec702, required?: true },
   { klass: Sva, required?: true },
   { klass: Vsoc, required?: true },

spec/controllers/dashboards_controller_spec.rb

@@ -198,7 +198,7 @@ def load_table(klass)
       stub_const('Scorecard::POPULATE_SUCCESS_MESSAGE', nil)
 
       get(:api_fetch, params: { csv_type: Scorecard.name })
-      expect(flash.notice).to eq("#{Scorecard.name} finished fetching data from it's api")
+      expect(flash.notice).to eq("#{Scorecard.name} finished fetching data from its api")
     end
 
     it 'displays error message' do

spec/utilities/scorecard_api/service_spec.rb

@@ -22,7 +22,7 @@
         allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
         allow(client_instance).to receive(:schools).and_return(response)
 
-        results = described_class.populate('scorecard')
+        results = described_class.populate
 
         expect(results.size).to eq(response_results.size * 2)
         expect(results).to all(be_a(Scorecard))
@@ -42,74 +42,11 @@
         allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
         allow(client_instance).to receive(:schools).and_return(response)
 
-        results = described_class.populate('scorecard')
+        results = described_class.populate
 
         expect(results.size).to eq(response_results.size)
         expect(results).to all(be_a(Scorecard))
       end
     end
   end
-
-  describe 'populate scorecard degree programs' do
-    let(:result_1) do
-      {
-        'latest.programs.cip_4_digit': [{
-          unitid: 1,
-          school: {},
-          credential: {}
-        }]
-      }
-    end
-    let(:result_2) do
-      {
-        'latest.programs.cip_4_digit': [{
-          unitid: 2,
-          school: {},
-          credential: {}
-        }]
-      }
-    end
-
-    let(:response_results) { [result_1, result_2] }
-    let(:client_instance) { instance_double(ScorecardApi::Client) }
-
-    context 'when total is greater than MAX_PAGE_SIZE' do
-      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE + 50 }
-      let(:body) { { results: response_results, metadata: { total: total } } }
-      let(:response) do
-        response = Faraday::Env.new
-        response[:body] = body
-        response
-      end
-
-      it 'calls ScorecardApi::Client twice' do
-        allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
-        allow(client_instance).to receive(:schools).and_return(response)
-
-        results = described_class.populate('scorecard degree program')
-
-        expect(results.size).to eq(response_results.size * 2)
-        expect(results).to all(be_a(ScorecardDegreeProgram))
-      end
-    end
-
-    context 'when total is less than MAX_PAGE_SIZE' do
-      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE - 50 }
-      let(:body) { { results: response_results, metadata: { total: total } } }
-      let(:response) do
-        response = Faraday::Env.new
-        response[:body] = body
-        response
-      end
-
-      it 'calls ScorecardApi::Client once' do
-        allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
-        allow(client_instance).to receive(:schools).and_return(response)
-
-        results = described_class.populate('scorecard degree program')
-        expect(results.size).to eq(response_results.size)
-        expect(results).to all(be_a(ScorecardDegreeProgram))
-      end
-    end
-  end
 end

spec/utilities/scorecard_degree_program_api/service_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ScorecardDegreeProgramApi::Service do
+  describe 'populate scorecard degree programs' do
+    let(:result_1) do
+      {
+        'latest.programs.cip_4_digit': [{
+          unitid: 1,
+          school: {},
+          credential: {}
+        }]
+      }
+    end
+    let(:result_2) do
+      {
+        'latest.programs.cip_4_digit': [{
+          unitid: 2,
+          school: {},
+          credential: {}
+        }]
+      }
+    end
+
+    let(:response_results) { [result_1, result_2] }
+    let(:client_instance) { instance_double(ScorecardApi::Client) }
+
+    context 'when total is greater than MAX_PAGE_SIZE' do
+      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE + 50 }
+      let(:body) { { results: response_results, metadata: { total: total } } }
+      let(:response) do
+        response = Faraday::Env.new
+        response[:body] = body
+        response
+      end
+
+      it 'calls ScorecardApi::Client twice' do
+        allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
+        allow(client_instance).to receive(:schools).and_return(response)
+
+        results = described_class.populate
+
+        expect(results.size).to eq(response_results.size * 2)
+        expect(results).to all(be_a(ScorecardDegreeProgram))
+      end
+    end
+
+    context 'when total is less than MAX_PAGE_SIZE' do
+      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE - 50 }
+      let(:body) { { results: response_results, metadata: { total: total } } }
+      let(:response) do
+        response = Faraday::Env.new
+        response[:body] = body
+        response
+      end
+
+      it 'calls ScorecardApi::Client once' do
+        allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
+        allow(client_instance).to receive(:schools).and_return(response)
+
+        results = described_class.populate
+        expect(results.size).to eq(response_results.size)
+        expect(results).to all(be_a(ScorecardDegreeProgram))
+      end
+    end
+  end
+end