From 0deb1f10f1e859aa88acfd9ccdab9860693be0d8 Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Tue, 1 Aug 2023 17:19:19 +0100 Subject: [PATCH 1/6] Match CV data to returns data for 2PT billing https://eaflood.atlassian.net/browse/WATER-4046 First iteration of mathcing returns to charge versions. This covers the first example which is a one-to-one match. We also have updated the results to include charge versions, charge elements and charge purposes. We intend to refine this in future iterations but we are adding them to confirm the matching algorithm is working correctly. --- app/models/legacy-base.model.js | 1 + app/models/returns/return.model.js | 24 +++++++++++++++++ app/models/returns/returns-base.model.js | 16 +++++++++++ app/services/check/two-part.service.js | 34 +++++++++++++++--------- 4 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 app/models/returns/return.model.js create mode 100644 app/models/returns/returns-base.model.js diff --git a/app/models/legacy-base.model.js b/app/models/legacy-base.model.js index 22adbf8619..43e6e553d6 100644 --- a/app/models/legacy-base.model.js +++ b/app/models/legacy-base.model.js @@ -24,6 +24,7 @@ class LegacyBaseModel extends BaseModel { const currentPath = __dirname return [ currentPath, + path.join(currentPath, 'returns'), path.join(currentPath, 'water'), path.join(currentPath, 'crm-v2') ] diff --git a/app/models/returns/return.model.js b/app/models/returns/return.model.js new file mode 100644 index 0000000000..0b6b699c6b --- /dev/null +++ b/app/models/returns/return.model.js @@ -0,0 +1,24 @@ +'use strict' + +/** + * Model for return + * @module ReturnModel + */ + +const ReturnsBaseModel = require('./returns-base.model.js') + +class ReturnModel extends ReturnsBaseModel { + static get tableName () { + return 'returns' + } + + static get idColumn () { + return 'returnId' + } + + static get translations () { + return [] + } +} + +module.exports = ReturnModel diff --git a/app/models/returns/returns-base.model.js b/app/models/returns/returns-base.model.js new file mode 100644 index 0000000000..e5584daae1 --- /dev/null +++ b/app/models/returns/returns-base.model.js @@ -0,0 +1,16 @@ +'use strict' + +/** + * Base class for all models based on the legacy 'returns' schema + * @module ReturnsBaseModel + */ + +const LegacyBaseModel = require('../legacy-base.model.js') + +class ReturnsBaseModel extends LegacyBaseModel { + static get schema () { + return 'returns' + } +} + +module.exports = ReturnsBaseModel diff --git a/app/services/check/two-part.service.js b/app/services/check/two-part.service.js index 1117ed7e2b..b94cae5b98 100644 --- a/app/services/check/two-part.service.js +++ b/app/services/check/two-part.service.js @@ -2,29 +2,39 @@ /** * Used to test the Two-part tariff matching logic - * @module SupplementaryDataService + * @module TwoPartService */ -const ChargeVersion = require('../../models/water/charge-version.model.js') const DetermineBillingPeriodsService = require('../billing/determine-billing-periods.service.js') -const FetchRegionService = require('../billing/fetch-region.service.js') +const LicenceModel = require('../../models/water/licence.model.js') +const ReturnModel = require('../../models/returns/return.model.js') async function go (naldRegionId) { - const region = await FetchRegionService.go(naldRegionId) const billingPeriods = DetermineBillingPeriodsService.go() const billingPeriod = billingPeriods[1] - const licenceRefs = await ChargeVersion.query() - .select('licenceRef') - .innerJoinRelated('chargeElements') - .where('regionCode', naldRegionId) + const licences = await LicenceModel.query() + .select('licences.licenceRef', 'licences.licenceId') + .innerJoinRelated('chargeVersions.chargeElements') + .where('chargeVersions.regionCode', naldRegionId) .where('chargeVersions.scheme', 'sroc') - .where('startDate', '<=', billingPeriod.endDate) - .where('isSection127AgreementEnabled', true) - .whereNot('status', 'draft') + .where('chargeVersions.startDate', '<=', billingPeriod.endDate) + .where('chargeVersions:chargeElements.isSection127AgreementEnabled', true) + .whereNot('chargeVersions.status', 'draft') + .withGraphFetched('chargeVersions.chargeElements.chargePurposes') - return [region.name, billingPeriod, licenceRefs] + for (const licence of licences) { + licence.returns = await ReturnModel.query() + .select('returnId', 'returnRequirement', 'startDate', 'endDate') + .where('licenceRef', licence.licenceRef) + // used in the service to filter out old returns here `src/lib/services/returns/api-connector.js` + .where('startDate', '>=', '2008-04-01') + .where('startDate', '<=', billingPeriod.endDate) + .where('endDate', '>=', billingPeriod.startDate) + } + + return licences } module.exports = { From 8c81c6f0e7b69a1bce5b445bc3a2d8f400dcf3c2 Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Wed, 2 Aug 2023 14:07:41 +0100 Subject: [PATCH 2/6] Add jsonAttributes to return model --- app/models/returns/return.model.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/returns/return.model.js b/app/models/returns/return.model.js index 0b6b699c6b..a72e2bf3d2 100644 --- a/app/models/returns/return.model.js +++ b/app/models/returns/return.model.js @@ -19,6 +19,13 @@ class ReturnModel extends ReturnsBaseModel { static get translations () { return [] } + + // Defining which fields contain json allows us to insert an object without needing to stringify it first + static get jsonAttributes () { + return [ + 'metadata' + ] + } } module.exports = ReturnModel From cbab0e6343b20517c26413ee20b244d1ae145011 Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Wed, 2 Aug 2023 15:46:21 +0100 Subject: [PATCH 3/6] Add migrations for returns.returns --- .../20230802134043_create_returns_schema.js | 13 ++++++ .../20230802134153_create_returns_returns.js | 40 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 db/migrations/20230802134043_create_returns_schema.js create mode 100644 db/migrations/20230802134153_create_returns_returns.js diff --git a/db/migrations/20230802134043_create_returns_schema.js b/db/migrations/20230802134043_create_returns_schema.js new file mode 100644 index 0000000000..5a8fe7f46f --- /dev/null +++ b/db/migrations/20230802134043_create_returns_schema.js @@ -0,0 +1,13 @@ +'use strict' + +exports.up = function (knex) { + return knex.raw(` + CREATE SCHEMA IF NOT EXISTS "returns"; + `) +} + +exports.down = function (knex) { + return knex.raw(` + DROP SCHEMA IF EXISTS "returns"; + `) +} diff --git a/db/migrations/20230802134153_create_returns_returns.js b/db/migrations/20230802134153_create_returns_returns.js new file mode 100644 index 0000000000..8fe5463dfa --- /dev/null +++ b/db/migrations/20230802134153_create_returns_returns.js @@ -0,0 +1,40 @@ +'use strict' + +const tableName = 'returns' + +exports.up = async function (knex) { + await knex + .schema + .withSchema('returns') + .createTable(tableName, table => { + // Primary Key + table.string('return_id').primary() + + // Data + table.string('regime') + table.string('licence_type') + table.string('licence_ref') + table.date('start_date') + table.date('end_date') + table.string('returns_frequency') + table.string('status') + table.string('source') + table.jsonb('metadata') + table.timestamp('created_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) + table.timestamp('updated_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) + table.date('received_date') + table.string('return_requirement') + table.date('due_date') + table.boolean('under_query') + table.string('under_query_comment') + table.boolean('is_test') + table.uuid('return_cycle_id') + }) +} + +exports.down = function (knex) { + return knex + .schema + .withSchema('returns') + .dropTableIfExists(tableName) +} From 4c39cbacfca9e417078fcbacb22a19e989262b59 Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Wed, 2 Aug 2023 15:46:52 +0100 Subject: [PATCH 4/6] Update helper to clear returns scema --- test/support/helpers/database.helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/helpers/database.helper.js b/test/support/helpers/database.helper.js index c79a26db07..1b795f52c4 100644 --- a/test/support/helpers/database.helper.js +++ b/test/support/helpers/database.helper.js @@ -19,7 +19,7 @@ const { db } = require('../../../db/db.js') * identity columns. For example, if a table relies on an incrementing ID the query will reset that to 1. */ async function clean () { - const schemas = ['water'] + const schemas = ['water', 'returns'] for (const schema of schemas) { const tables = await _tableNames(schema) From 7eb03b7adf1cbbfe44ed82f1fcb41a47d080da3f Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Wed, 2 Aug 2023 15:47:13 +0100 Subject: [PATCH 5/6] Create return model test and helper --- test/models/returns/return.model.test.js | 34 +++++++++ test/support/helpers/returns/return.helper.js | 74 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 test/models/returns/return.model.test.js create mode 100644 test/support/helpers/returns/return.helper.js diff --git a/test/models/returns/return.model.test.js b/test/models/returns/return.model.test.js new file mode 100644 index 0000000000..b33015d4f1 --- /dev/null +++ b/test/models/returns/return.model.test.js @@ -0,0 +1,34 @@ +'use strict' + +// Test framework dependencies +const Lab = require('@hapi/lab') +const Code = require('@hapi/code') + +const { describe, it, beforeEach } = exports.lab = Lab.script() +const { expect } = Code + +// Test helpers +const DatabaseHelper = require('../../support/helpers/database.helper.js') +const ReturnHelper = require('../../support/helpers/returns/return.helper.js') + +// Thing under test +const ReturnModel = require('../../../app/models/returns/return.model.js') + +describe('Return model', () => { + let testRecord + + beforeEach(async () => { + await DatabaseHelper.clean() + + testRecord = await ReturnHelper.add() + }) + + describe('Basic query', () => { + it('can successfully run a basic query', async () => { + const result = await ReturnModel.query().findById(testRecord.returnId) + + expect(result).to.be.an.instanceOf(ReturnModel) + expect(result.returnId).to.equal(testRecord.returnId) + }) + }) +}) diff --git a/test/support/helpers/returns/return.helper.js b/test/support/helpers/returns/return.helper.js new file mode 100644 index 0000000000..245199f3e8 --- /dev/null +++ b/test/support/helpers/returns/return.helper.js @@ -0,0 +1,74 @@ +'use strict' + +/** + * @module ReturnHelper + */ + +const ReturnModel = require('../../../../app/models/returns/return.model.js') + +/** + * Add a new return + * + * If no `data` is provided, default values will be used. These are + * + * - `returnId` - v1:1:9/99/99/99/9999:10021668:2022-04-01:2023-03-31 + * - `regime` - water + * - `licenceType` - abstraction + * - `startDate` - 2022-04-01 + * - `endDate` - 2023-03-31 + * - `returnsFrequency` - month + * - `status` - completed + * - `source` - NALD + * - `metadata` - {} + * - `receivedDate` - 2023-04-12 + * - `returnRequirement` - 99999 + * - `dueDate` - 2023-04-28 + * - `returnCycleId` - 2eb314fe-da45-4ae9-b418-7d89a8c49c51 + * + * @param {Object} [data] Any data you want to use instead of the defaults used here or in the database + * + * @returns {module:ReturnModel} The instance of the newly created record + */ +function add (data = {}) { + const insertData = defaults(data) + + return ReturnModel.query() + .insert({ ...insertData }) + .returning('*') +} + +/** + * Returns the defaults used when creating a new return + * + * It will override or append to them any data provided. Mainly used by the `add()` method, we make it available + * for use in tests to avoid having to duplicate values. + * + * @param {Object} [data] Any data you want to use instead of the defaults used here or in the database + */ +function defaults (data = {}) { + const defaults = { + returnId: 'v1:1:9/99/99/99/9999:10021668:2022-04-01:2023-03-31', + regime: 'water', + licenceType: 'abstraction', + startDate: '2022-04-01', + endDate: '2023-03-31', + returnsFrequency: 'month', + status: 'completed', + source: 'NALD', + metadata: {}, + receivedDate: '2023-04-12', + returnRequirement: '99999', + dueDate: '2023-04-28', + returnCycleId: '2eb314fe-da45-4ae9-b418-7d89a8c49c51' + } + + return { + ...defaults, + ...data + } +} + +module.exports = { + add, + defaults +} From 94a520be4f79bdba565123b951eb789483923adf Mon Sep 17 00:00:00 2001 From: Jason Claxton Date: Wed, 2 Aug 2023 16:01:24 +0100 Subject: [PATCH 6/6] Update timestamps --- db/migrations/20230802134153_create_returns_returns.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db/migrations/20230802134153_create_returns_returns.js b/db/migrations/20230802134153_create_returns_returns.js index 8fe5463dfa..38ef567d73 100644 --- a/db/migrations/20230802134153_create_returns_returns.js +++ b/db/migrations/20230802134153_create_returns_returns.js @@ -20,8 +20,6 @@ exports.up = async function (knex) { table.string('status') table.string('source') table.jsonb('metadata') - table.timestamp('created_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) - table.timestamp('updated_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) table.date('received_date') table.string('return_requirement') table.date('due_date') @@ -29,6 +27,10 @@ exports.up = async function (knex) { table.string('under_query_comment') table.boolean('is_test') table.uuid('return_cycle_id') + + // Legacy timestamps + table.timestamp('created_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) + table.timestamp('updated_at', { useTz: false }).notNullable().defaultTo(knex.fn.now()) }) }