Merge pull request #133 from slovensko-digital/ECO-253/update_upvs_da…

…taset_source

ECO-253 Update fetching UPVS datasets

app/jobs/upvs/fetch_public_authority_active_edesks_list_job.rb

@@ -1,8 +1,10 @@
 class Upvs::FetchPublicAuthorityActiveEdesksListJob < ApplicationJob
   queue_as :upvs
 
-  def perform(url, downloader: HarvesterUtils::Downloader)
-    csv_file = downloader.download_file(url)
+  DATASET_URL = 'https://data.slovensko.sk/download?id=c67849ad-bd97-4c43-90d4-42f00f450c56'
+
+  def perform(downloader: HarvesterUtils::Downloader)
+    csv_file = downloader.download_file(DATASET_URL)
     csv_options = { col_sep: File.open(csv_file) { |f| f.readline }.include?(';') ? ';' : ',', headers: true }
 
     TemporaryPublicAuthorityActiveEdesk.transaction do

app/jobs/upvs/fetch_public_authority_edesks_list_job.rb

@@ -1,8 +1,10 @@
 class Upvs::FetchPublicAuthorityEdesksListJob < ApplicationJob
   queue_as :upvs
 
-  def perform(url, downloader: HarvesterUtils::Downloader)
-    csv_file = downloader.download_file(url)
+  DATASET_URL = 'https://data.slovensko.sk/download?id=ce7c38ed-31e2-423a-b963-6bfda7696666'
+
+  def perform(downloader: HarvesterUtils::Downloader)
+    csv_file = downloader.download_file(DATASET_URL)
     csv_options = { col_sep: File.open(csv_file) { |f| f.readline }.include?(';') ? ';' : ',', headers: true }
 
     TemporaryPublicAuthorityEdesk.transaction do

app/jobs/upvs/fetch_services_with_forms_list_job.rb

@@ -1,8 +1,10 @@
 class Upvs::FetchServicesWithFormsListJob < ApplicationJob
   queue_as :upvs
 
-  def perform(url, downloader: HarvesterUtils::Downloader)
-    zip_file = downloader.download_file(url)
+  DATASET_URL = 'https://data.slovensko.sk/download?id=13b52968-e3d5-4d75-b8f1-fa9f25789033'
+
+  def perform(downloader: HarvesterUtils::Downloader)
+    zip_file = downloader.download_file(DATASET_URL)
     csv_file = downloader.extract_csv(zip_file)
 
     csv_options = {

lib/tasks/upvs.rake

@@ -1,16 +1,16 @@
 namespace :upvs do
   desc 'Sync all public authority eDesks'
   task 'public_authority_edesks:sync' => :environment do
-    Upvs::FindPublicAuthorityEdeskListJob.perform_later
+    Upvs::FetchPublicAuthorityEdesksListJob.perform_later
   end
 
   desc 'Sync public authority eDesks'
   task 'public_authority_active_edesks:sync' => :environment do
-    Upvs::FindPublicAuthorityActiveEdeskListJob.perform_later
+    Upvs::FetchPublicAuthorityActiveEdesksListJob.perform_later
   end
 
   desc 'Sync services with forms list'
   task 'services_with_forms:sync' => :environment do
-    Upvs::FindServicesWithFormsListJob.perform_later
+    Upvs::FetchServicesWithFormsListJob.perform_later
   end
 end

spec/jobs/upvs/fetch_public_authority_active_edesks_list_job_spec.rb

@@ -2,14 +2,14 @@
 
 RSpec.describe Upvs::FetchPublicAuthorityActiveEdesksListJob, type: :job do
   describe '#perform' do
-    let(:url) { 'https://example.com' }
+    let(:url) { 'https://data.slovensko.sk/download?id=c67849ad-bd97-4c43-90d4-42f00f450c56' }
 
     let(:downloader) { double }
 
     it 'downloads and imports public authority eDesks in V1 format' do
       expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1.csv'))
 
-      subject.perform(url, downloader: downloader)
+      subject.perform(downloader: downloader)
 
       expect(Upvs::PublicAuthorityActiveEdesk.first).to have_attributes(
         uri: 'ico://sk/00332674',
@@ -27,7 +27,7 @@
         expect(Upvs::PublicAuthorityActiveEdesk.count).to eq(10)
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1.csv'))
 
-        subject.perform(url, downloader: downloader)
+        subject.perform(downloader: downloader)
 
         expect(Upvs::PublicAuthorityActiveEdesk.count).to eq(7)
       end
@@ -37,7 +37,7 @@
       it 'does not import public authority eDesks' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error(RuntimeError)
+        expect { subject.perform(downloader: downloader) }.to raise_error(RuntimeError)
 
         expect(Upvs::PublicAuthorityActiveEdesk.count).to eq(0)
       end
@@ -48,22 +48,22 @@
         expect(Upvs::PublicAuthorityActiveEdesk.count).to eq(10)
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error(RuntimeError)
+        expect { subject.perform(downloader: downloader) }.to raise_error(RuntimeError)
 
         expect(Upvs::PublicAuthorityActiveEdesk.count).to eq(10)
       end
 
       it 'raises custom error' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error('ico://sk/99166260 does not match 166260')
+        expect { subject.perform(downloader: downloader) }.to raise_error('ico://sk/99166260 does not match 166260')
       end
 
       it 'does not raise custom error if only leading zeros difference' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1-missing-leading-zeros.csv'))
 
-        # subject.perform(url, downloader: downloader)
-        expect { subject.perform(url, downloader: downloader) }.not_to raise_error
+        # subject.perform(downloader: downloader)
+        expect { subject.perform(downloader: downloader) }.not_to raise_error
 
         expect(Upvs::PublicAuthorityActiveEdesk.last).to have_attributes(
           uri: 'ico://sk/214973_10001',
@@ -77,7 +77,7 @@
       it 'raises custom error' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/active-edesks-v1-incorrect-encoding.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error('Incorrect encoding')
+        expect { subject.perform(downloader: downloader) }.to raise_error('Incorrect encoding')
       end
     end
   end

spec/jobs/upvs/fetch_public_authority_edesks_list_job_spec.rb

@@ -2,14 +2,14 @@
 
 RSpec.describe Upvs::FetchPublicAuthorityEdesksListJob, type: :job do
   describe '#perform' do
-    let(:url) { 'https://example.com' }
+    let(:url) { 'https://data.slovensko.sk/download?id=ce7c38ed-31e2-423a-b963-6bfda7696666' }
 
     let(:downloader) { double }
 
     it 'downloads and imports all public authority eDesks' do
       expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks.csv'))
 
-      subject.perform(url, downloader: downloader)
+      subject.perform(downloader: downloader)
 
       expect(Upvs::PublicAuthorityEdesk.first).to have_attributes(
         uri: 'ico://sk/00332674',
@@ -27,7 +27,7 @@
         expect(Upvs::PublicAuthorityEdesk.count).to eq(10)
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks.csv'))
 
-        subject.perform(url, downloader: downloader)
+        subject.perform(downloader: downloader)
 
         expect(Upvs::PublicAuthorityEdesk.count).to eq(7)
       end
@@ -37,7 +37,7 @@
       it 'does not import public authority eDesks' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error(RuntimeError)
+        expect { subject.perform(downloader: downloader) }.to raise_error(RuntimeError)
 
         expect(Upvs::PublicAuthorityEdesk.count).to eq(0)
       end
@@ -48,21 +48,21 @@
         expect(Upvs::PublicAuthorityEdesk.count).to eq(10)
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error(RuntimeError)
+        expect { subject.perform(downloader: downloader) }.to raise_error(RuntimeError)
 
         expect(Upvs::PublicAuthorityEdesk.count).to eq(10)
       end
 
       it 'raises custom error' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks-not-matching.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error('ico://sk/99166260 does not match 166260')
+        expect { subject.perform(downloader: downloader) }.to raise_error('ico://sk/99166260 does not match 166260')
       end
 
       it 'does not raise custom error if only leading zeros difference' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks-missing-leading-zeros.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.not_to raise_error
+        expect { subject.perform(downloader: downloader) }.not_to raise_error
 
         expect(Upvs::PublicAuthorityEdesk.last).to have_attributes(
           uri: 'ico://sk/214973_10001',
@@ -76,7 +76,7 @@
       it 'raises custom error' do
         expect(downloader).to receive(:download_file).with(url).and_return(fixture_filepath('upvs/edesks-incorrect-encoding.csv'))
 
-        expect { subject.perform(url, downloader: downloader) }.to raise_error('Incorrect encoding')
+        expect { subject.perform(downloader: downloader) }.to raise_error('Incorrect encoding')
       end
     end
   end

spec/jobs/upvs/fetch_services_with_forms_list_job_spec.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe Upvs::FetchServicesWithFormsListJob, type: :job do
   describe '#perform' do
-    let(:url) { 'https://example.com' }
+    let(:url) { 'https://data.slovensko.sk/download?id=13b52968-e3d5-4d75-b8f1-fa9f25789033' }
 
     let(:downloader) { double }
 
@@ -12,7 +12,7 @@
         HarvesterUtils::Downloader.extract_csv(zip_file)
       end
 
-      subject.perform(url, downloader: downloader)
+      subject.perform(downloader: downloader)
 
       expect(Upvs::ServiceWithForm.first).to have_attributes(
         instance_id: 2083,
@@ -36,7 +36,7 @@
         HarvesterUtils::Downloader.extract_csv(zip_file)
       end
 
-      subject.perform(url, downloader: downloader)
+      subject.perform(downloader: downloader)
 
       expect(Upvs::ServiceWithForm.first).to have_attributes(
         instance_id: 2082,
@@ -60,7 +60,7 @@
         HarvesterUtils::Downloader.extract_csv(zip_file)
       end
 
-      subject.perform(url, downloader: downloader)
+      subject.perform(downloader: downloader)
 
       expect(Upvs::ServiceWithForm.first).to have_attributes(
         instance_id: 2088,
@@ -85,7 +85,7 @@
           HarvesterUtils::Downloader.extract_csv(zip_file)
         end
 
-        subject.perform(url, downloader: downloader)
+        subject.perform(downloader: downloader)
 
         expect(Upvs::ServiceWithForm.first).to have_attributes(
           instance_id: 29644,
@@ -114,7 +114,7 @@
           HarvesterUtils::Downloader.extract_csv(zip_file)
         end
 
-        subject.perform(url, downloader: downloader)
+        subject.perform(downloader: downloader)
 
         expect(Upvs::ServiceWithForm.count).to eq(20)
       end