Add rake task to sync using AusPayNet API (#61)

* Get something working

* End to end test

* Split bsb tasks into two tasks

* Small random cleanup

* Generate latest update json

* Rubocop

* Use LEADER_WIDTH to calc outputparam width

* Extract output param width

* Add README section

* Misc cleanup

* Fixup README

* Add unit test for aus pay net api client

* fix: update bank list

Signed-off-by: David Looi <david@coinjar.com>

* fix: typo

Signed-off-by: David Looi <david@coinjar.com>

---------

Signed-off-by: David Looi <david@coinjar.com>
Co-authored-by: David Looi <david@coinjar.com>

Gemfile

@@ -6,6 +6,9 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'bundler', '~> 2.0'
+gem 'faraday'
+gem 'minitest-stub-const'
 gem 'net-ftp', '~> 0.1.3'
 gem 'rake', '~> 13.0'
 gem 'rubocop'
+gem 'vcr'

README.md

@@ -68,18 +68,53 @@ Two data sources are used:
 
 Other formats of APCA BSB data is available from http://bsb.apca.com.au.
 
-## Update source
+## Update BSB Bank List
 
 At the moment BSB data is a manual download from the Auspaynet site [here](https://bsb.auspaynet.com.au/).
 
-You will need to download two files, place them in `tmp/`:
+You will need to download the Key to Abbreviations and BSB Number file and place it in `tmp/`:
 - `Reference Documents` button > `Key to Abbreviations and BSB Number` in CSV format.
+
+Run the sync task with the files to complete sync of the latest data:
+
+```sh
+rake bsb:sync_bank_list['tmp/key to abbreviations and bsb numbers (august 2024).csv']
+```
+
+Browse the list of database changes, make a few queries on the website to ensure the results are the same.
+
+## Update BSB DB
+
+`config/bsb_db.json` can be updated by running the `bsb:sync_bsb_db` rake task.
+This task depends on you having you having an Aus Pay Net API subscription and key and that key being available in the
+`AUSPAYNET_SUB_KEY` environment variable.
+
+You can apply for an API subscription and key by visiting [AusPayNet's bsb page](https://bsb.auspaynet.com.au/),
+clicking the `API Registration` button and following the prompts.
+
+Once you have a key, the task can be run as follows
+
+```sh
+AUSPAYNET_SUB_KEY="your_key_here" rake bsb:sync_bsb_db
+```
+
+This will update the `config/bsb_db.json` file with the latest information and will produce a
+`config/latest_update.json` file that contains a breakdown of additions, deletions and modifications to make spot
+checking results easier.
+
+Browse the list of database changes, make a few queries on the website to ensure the results are the same.
+
+## Update BSB DB (Manual)
+
+BSB DB data can be downloaded manually from the Auspaynet site [here](https://bsb.auspaynet.com.au/).
+
+You will need to download the BSB directory and place it in `tmp/`:
 - `Download BSB Files` button > `BSB Directory (Full Version)` in TEXT format.
 
 Run the sync task with the files to complete sync of the latest data:
 
 ```sh
-rake bsb:sync['tmp/key to abbreviations and bsb numbers (august 2024).csv','tmp/BSBDirectoryAug24-341.txt']
+rake bsb:sync_bsb_db_manual['tmp/BSBDirectoryAug24-341.txt']
 ```
 
 Browse the list of database changes, make a few queries on the website to ensure the results are the same.

Rakefile

@@ -2,6 +2,7 @@
 
 require 'bundler/gem_tasks'
 load 'lib/tasks/bsb_tasks.rake'
+load 'lib/tasks/sync_bsb_db.rake'
 
 require 'rake/testtask'
 

lib/bsb.rb

@@ -5,6 +5,8 @@
 require 'bsb_number_validator'
 
 module BSB
+  DB_FILEPATH = 'config/bsb_db.json'
+  CHANGES_FILEPATH = 'config/latest_update.json'
   class << self
     def lookup(number)
       bsb = normalize(number)
@@ -42,7 +44,7 @@ def normalize(str)
     protected
 
     def data_hash
-      @data_hash ||= JSON.parse(File.read(File.expand_path('../config/bsb_db.json', __dir__)))
+      @data_hash ||= JSON.parse(File.read(File.expand_path("../#{DB_FILEPATH}", __dir__)))
     end
 
     def bank_list

lib/bsb/aus_pay_net/client.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module BSB
+  module AusPayNet
+    module Client
+      class MissingSubscriptionKeyError < StandardError; end
+
+      OUTPUT_PARAM_WIDTH = 30
+      LEADER_WIDTH = OUTPUT_PARAM_WIDTH + 11
+
+      Response = Struct.new(:body, keyword_init: true)
+
+      def self.fetch_all_bsbs
+        subscription_key = ENV.fetch('AUSPAYNET_SUB_KEY', nil)
+        if subscription_key.nil?
+          raise MissingSubscriptionKeyError, "the environment variable 'AUSPAYNET_SUB_KEY' must be present"
+        end
+
+        conn = Faraday.new(
+          url: 'https://auspaynet-bicbsb-api-prod.azure-api.net',
+          headers: {
+            'Content-Type': 'application/json',
+            'Ocp-Apim-Subscription-Key': subscription_key
+          }
+        ) do |faraday|
+          faraday.response :raise_error
+        end
+
+        response = conn.post('/bsbquery/manual/paths/invoke') do |req|
+          req.body = { outputparam: ' ' * OUTPUT_PARAM_WIDTH }.to_json
+        end
+
+        Response.new(body: response.body[LEADER_WIDTH..])
+      end
+    end
+  end
+end

lib/bsb/base_generator.rb

@@ -4,6 +4,8 @@
 
 module BSB
   class BaseGenerator
+    attr_reader :hash
+
     def initialize(hash)
       @hash = hash
     end

lib/bsb/database_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'bsb/base_generator'
+require 'bsb/aus_pay_net/client'
 
 module BSB
   class DatabaseGenerator < BaseGenerator
@@ -16,5 +17,24 @@ def self.load_file(filename)
       end
       new(hash)
     end
+
+    def self.fetch_latest
+      response = BSB::AusPayNet::Client.fetch_all_bsbs
+
+      hash = {}
+      JSON.parse(response.body).each do |bsb_config|
+        bsb = bsb_config.fetch('BSBCode').delete('-')
+        hash[bsb] = [
+          bsb_config.fetch('FiMnemonic'),
+          bsb_config.fetch('BSBName'),
+          bsb_config.fetch('Address'),
+          bsb_config.fetch('Suburb'),
+          bsb_config.fetch('State'),
+          bsb_config.fetch('Postcode'),
+          'PEH'.chars.map { bsb_config.fetch('StreamCode').include?(_1) ? _1 : ' ' }.join
+        ]
+      end
+      new(hash)
+    end
   end
 end

lib/tasks/bsb_tasks.rake

@@ -1,26 +1,30 @@
 # frozen_string_literal: true
 
+require 'bsb'
+require 'bsb/database_generator'
+require 'bsb/bank_list_generator'
+
 namespace :bsb do
   desc 'Sync config/*.json.'
-  task :sync, [:keyfile, :bsbfile] do |_t, args|
-    require 'bsb/base_generator'
-    bank_list_filename = args[:keyfile]
+  task :sync_bsb_db_manual, [:bsbfile] do |_t, args|
     db_list_filename = args[:bsbfile]
 
+    if db_list_filename
+      bsb_db_gen = BSB::DatabaseGenerator.load_file(db_list_filename)
+      File.write(BSB::DB_FILEPATH, bsb_db_gen.json)
+    else
+      warn 'Missing bsb db "BSBDirectory"'
+    end
+  end
+
+  task :sync_bank_list, [:keyfile] do |_t, args|
+    bank_list_filename = args[:keyfile]
+
     if bank_list_filename
-      require 'bsb/bank_list_generator'
       bsb_bl_gen = BSB::BankListGenerator.load_file(bank_list_filename)
       File.write('config/bsb_bank_list.json', bsb_bl_gen.json)
     else
       warn 'Missing bank list "KEY TO ABBREVIATIONS AND BSB NUMBERS"'
     end
-
-    if db_list_filename
-      require 'bsb/database_generator'
-      bsb_db_gen = BSB::DatabaseGenerator.load_file(db_list_filename)
-      File.write('config/bsb_db.json', bsb_db_gen.json)
-    else
-      warn 'Missing bsb db "BSBDirectory"'
-    end
   end
 end

lib/tasks/sync_bsb_db.rake

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'bsb'
+require 'bsb/database_generator'
+
+namespace :bsb do
+  desc 'Sync config/bsb_db.json with data provided by AusPayNet'
+  task :sync_bsb_db do
+    latest_db = BSB::DatabaseGenerator.fetch_latest
+    existing_db_hash = JSON.parse(File.read(BSB::DB_FILEPATH))
+    latest_db_hash = latest_db.hash
+
+    deletions = existing_db_hash.reject { |bsb, _| latest_db_hash.key?(bsb) }
+    additions = latest_db_hash.reject { |bsb, _| existing_db_hash.key?(bsb) }
+    modifications = {}
+
+    latest_db_hash.each do |bsb, data|
+      next unless existing_db_hash.key?(bsb) && existing_db_hash[bsb] != data
+
+      modifications[bsb] = data
+    end
+
+    changes_json = JSON.pretty_generate(
+      {
+        additions: additions,
+        deletions: deletions,
+        modifications: modifications
+      }
+    )
+
+    File.write(BSB::DB_FILEPATH, latest_db.json)
+    File.write(BSB::CHANGES_FILEPATH, changes_json)
+  end
+end

test/bsb/aus_pay_net/client_test.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'bsb/aus_pay_net/client'
+require 'test_helper'
+
+describe BSB::AusPayNet::Client do
+  describe '.fetch_all_bsbs' do
+    before { ENV['AUSPAYNET_SUB_KEY'] = 'something' }
+
+    it 'returns the expected response' do
+      VCR.use_cassette('auspaynet_fetch_all_bsbs') do
+        response = BSB::AusPayNet::Client.fetch_all_bsbs
+        assert_equal(response.class, BSB::AusPayNet::Client::Response)
+        assert_equal(response.body.class, String)
+        assert_equal(JSON.parse(response.body).count, 2)
+      end
+    end
+  end
+end

test/fixtures/bsb_db.json

@@ -0,0 +1,20 @@
+{
+  "333333": [
+    "AAA",
+    "Aviato3",
+    "123 Fakest Street",
+    "Canberra",
+    "ACT",
+    "1234",
+    "P H"
+  ],
+  "987654": [
+    "EST",
+    "Aviato2",
+    "123 Old Faker Street",
+    "Ballina",
+    "NSW",
+    "1234",
+    "P  "
+  ]
+}