From bf59a46d46821d4bf4f29b0009d662f08cf20de6 Mon Sep 17 00:00:00 2001 From: Kevin Leyow Date: Thu, 11 Jun 2020 06:51:20 -0500 Subject: [PATCH] edit ci and add seeds and add enpoints * Edited CI to build PISP docker image. * Addressed comments. Updated ci to python 3. (#744) Feature/335 thirdparty callbacks (#748) * Add new endpoint: `THIRDPARTY_CALLBACK_URL_TRX_REQ_POST` * cleanup * skip devDependencies in audit:check Update pisp/master (#755) * fix for python error in CI (#733) Co-authored-by: Shashi * Ensure 'timeout-reserved' notification action passes through and not converted to 'abort' action (#736) * Update CS shared (#737) * Feature/1332 enable on-us transfers (#738) * Added ENABLE_ON_US_TRANSFERS * Bumped up the version * Feature/otc 525 implement get transaction object by transfer (#735) * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id - Post ledger entry Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * OTC-525 Implement GET transaction Object by transferId Bumput up versions * OTC-525 Implement GET transaction Object by transferId Resolved dependency updates * OTC-525 Implement GET transaction Object by transferId * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * changed docker dependency in circle CI image scan from python-dev to python3-dev (#741) * Updated python in some other places in circle CI (#742) * Fix the image scan step in circle CI * Resolved audit checks * Feature/#1335 aborted on put (#740) * added error log if action REJECT comes into fulfil handler Co-authored-by: Valentin * Bugfix/deadlock on adjust limits (#745) * added unique index on participantLimit and logging * added unique index on participantLimit and logging * resolve audit issues * fixed coverage tests Co-authored-by: Shashi * Updated dependencies and product version for issue: https://github.com/mojaloop/project/issues/1378 (#747) * Update error message (#749) * Update error message when Payer FSP and Payee FSP are the same and on-us is not enabled.. (Added text "FSP" to specify) * Updated unit test * #1423: Bulk transfers error processing in Central Ledger (#743) * Updates for bulk error processing * Bump version * Updates for bulk transfer error processing * Updates for bulk transfer error processing * More updates for bulk error processing * changes to cater for bulk_abort * updated central-services-shared * Updates for bulk error processing * Add unit test for BULK_ABORT branch in transfer fulfil handler * Add unit test for BULK_ABORT branch in transfer facade * Small fix for position handler test for BULK_ABORT branch Co-authored-by: Rajiv Mothilal * Bump version for release (#750) * Feature/#1334 patch request notif (#751) * added handling of request for notification by payee functionality * improved coverage and added missing action letter Co-authored-by: Valentin * chore(package): update contributors list & deps * chore: audit & deps update Co-authored-by: shashi165 <33355509+shashi165@users.noreply.github.com> Co-authored-by: Shashi Co-authored-by: Steven Oderayi Co-authored-by: vijayg10 <33152110+vijayg10@users.noreply.github.com> Co-authored-by: lazolalucas Co-authored-by: Valentin Genev Co-authored-by: Valentin Co-authored-by: Adrian Enns Co-authored-by: Sam <10507686+elnyry-sam-k@users.noreply.github.com> Co-authored-by: Rajiv Mothilal chore: add thirdparty endpoints to database seeds (#779) * chore: add thirdparty endpoints to database seeds * chore: update dependencies for vulnerabilities * refactor: change name length to accomodate new endpoints * chore: sync package-lock * chore: remove migrations and shorten endpoint names * chore: fix find and replace error * chore: fix spelling refactor: update pisp/master (#781) * fix for python error in CI (#733) Co-authored-by: Shashi * Ensure 'timeout-reserved' notification action passes through and not converted to 'abort' action (#736) * Update CS shared (#737) * Feature/1332 enable on-us transfers (#738) * Added ENABLE_ON_US_TRANSFERS * Bumped up the version * Feature/otc 525 implement get transaction object by transfer (#735) * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id - Post ledger entry Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * OTC-525 Implement GET transaction Object by transferId Bumput up versions * OTC-525 Implement GET transaction Object by transferId Resolved dependency updates * OTC-525 Implement GET transaction Object by transferId * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * changed docker dependency in circle CI image scan from python-dev to python3-dev (#741) * Updated python in some other places in circle CI (#742) * Fix the image scan step in circle CI * Resolved audit checks * Feature/#1335 aborted on put (#740) * added error log if action REJECT comes into fulfil handler Co-authored-by: Valentin * Bugfix/deadlock on adjust limits (#745) * added unique index on participantLimit and logging * added unique index on participantLimit and logging * resolve audit issues * fixed coverage tests Co-authored-by: Shashi * Updated dependencies and product version for issue: https://github.com/mojaloop/project/issues/1378 (#747) * Update error message (#749) * Update error message when Payer FSP and Payee FSP are the same and on-us is not enabled.. (Added text "FSP" to specify) * Updated unit test * #1423: Bulk transfers error processing in Central Ledger (#743) * Updates for bulk error processing * Bump version * Updates for bulk transfer error processing * Updates for bulk transfer error processing * More updates for bulk error processing * changes to cater for bulk_abort * updated central-services-shared * Updates for bulk error processing * Add unit test for BULK_ABORT branch in transfer fulfil handler * Add unit test for BULK_ABORT branch in transfer facade * Small fix for position handler test for BULK_ABORT branch Co-authored-by: Rajiv Mothilal * Bump version for release (#750) * Feature/#1334 patch request notif (#751) * added handling of request for notification by payee functionality * improved coverage and added missing action letter Co-authored-by: Valentin * Feature/1468 bulk quotes endpoints (#761) * version change * Added FSPIOP_CALLBACK_URL_BULK_QUOTES endpoint to seeds and updated population scripts and created one for local legacy simulator updated dependencies * updated dependencies to resolve audit issues * Feature/#1375: GET bulk transfer implementation (#760) * Add bulk get topic and handler * Implement GET bulk transfer logic * Restore default config * Add unit tests * Bump version * More bug fixes * Fix unit test * ensure error code is returned as string and not number for bulk get * Add bulk get to handlers list for cli startup (#765) * Reset package-lock.json to fix bug with version update for AJV (#766) * SemVar fix (#767) * Fix error callback for bulk transfers REJECTED scenario (#768) * Correct FSPIOP API version for admin API (#769) * #1547: Ignore "RESERVED" transferState from v1.0 clients on fulfill callback (#770) * Ignore RESERVE transferState from v1.0 clients on fulfil callback * Update package.json Co-authored-by: Sam <10507686+elnyry-sam-k@users.noreply.github.com> Co-authored-by: Sam <10507686+elnyry-sam-k@users.noreply.github.com> * Bump version to v111.1.2 to fix broken release (#771) * #1547: Fail transfer fulfill with "RESERVED" state and v1.0 content-type (#773) * Update dependencies * Bump version * Fix integration tests * #1547: Update dependencies (central-object-store etc.) (#774) * Update dependencies * Bump version * fix: package.json & package-lock.json to reduce vulnerabilities (#775) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-LODASH-590103 * Fix bug in volumes of temp_curl service (#778) * Edited CI to build PISP docker image. (#734) * Edited CI to build PISP docker image. * Addressed comments. * Updated ci to python 3. (#744) * Update pisp/master (#755) * fix for python error in CI (#733) Co-authored-by: Shashi * Ensure 'timeout-reserved' notification action passes through and not converted to 'abort' action (#736) * Update CS shared (#737) * Feature/1332 enable on-us transfers (#738) * Added ENABLE_ON_US_TRANSFERS * Bumped up the version * Feature/otc 525 implement get transaction object by transfer (#735) * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id - Post ledger entry Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * OTC-525 Implement GET transaction Object by transferId Bumput up versions * OTC-525 Implement GET transaction Object by transferId Resolved dependency updates * OTC-525 Implement GET transaction Object by transferId * OTC-525 Implement GET transaction Object by transferId Changes: Updated swagger def to include new endpoints for: - Get transaction by transfer id Added new method for: - Get transaction by transfer id Added unit tests for Get transaction by transfer id * changed docker dependency in circle CI image scan from python-dev to python3-dev (#741) * Updated python in some other places in circle CI (#742) * Fix the image scan step in circle CI * Resolved audit checks * Feature/#1335 aborted on put (#740) * added error log if action REJECT comes into fulfil handler Co-authored-by: Valentin * Bugfix/deadlock on adjust limits (#745) * added unique index on participantLimit and logging * added unique index on participantLimit and logging * resolve audit issues * fixed coverage tests Co-authored-by: Shashi * Updated dependencies and product version for issue: https://github.com/mojaloop/project/issues/1378 (#747) * Update error message (#749) * Update error message when Payer FSP and Payee FSP are the same and on-us is not enabled.. (Added text "FSP" to specify) * Updated unit test * #1423: Bulk transfers error processing in Central Ledger (#743) * Updates for bulk error processing * Bump version * Updates for bulk transfer error processing * Updates for bulk transfer error processing * More updates for bulk error processing * changes to cater for bulk_abort * updated central-services-shared * Updates for bulk error processing * Add unit test for BULK_ABORT branch in transfer fulfil handler * Add unit test for BULK_ABORT branch in transfer facade * Small fix for position handler test for BULK_ABORT branch Co-authored-by: Rajiv Mothilal * Bump version for release (#750) * Feature/#1334 patch request notif (#751) * added handling of request for notification by payee functionality * improved coverage and added missing action letter Co-authored-by: Valentin * chore(package): update contributors list & deps * chore: audit & deps update Co-authored-by: shashi165 <33355509+shashi165@users.noreply.github.com> Co-authored-by: Shashi Co-authored-by: Steven Oderayi Co-authored-by: vijayg10 <33152110+vijayg10@users.noreply.github.com> Co-authored-by: lazolalucas Co-authored-by: Valentin Genev Co-authored-by: Valentin Co-authored-by: Adrian Enns Co-authored-by: Sam <10507686+elnyry-sam-k@users.noreply.github.com> Co-authored-by: Rajiv Mothilal * chore: add thirdparty endpoints to database seeds (#779) * chore: add thirdparty endpoints to database seeds * chore: update dependencies for vulnerabilities * refactor: change name length to accomodate new endpoints * chore: sync package-lock * chore: remove migrations and shorten endpoint names * chore: fix find and replace error * chore: fix spelling * chore: update packages * chore: sync package-lock Co-authored-by: shashi165 <33355509+shashi165@users.noreply.github.com> Co-authored-by: Shashi Co-authored-by: Steven Oderayi Co-authored-by: vijayg10 <33152110+vijayg10@users.noreply.github.com> Co-authored-by: lazolalucas Co-authored-by: Valentin Genev Co-authored-by: Valentin Co-authored-by: Adrian Enns Co-authored-by: Sam <10507686+elnyry-sam-k@users.noreply.github.com> Co-authored-by: Rajiv Mothilal Co-authored-by: Snyk bot Co-authored-by: Ali Behnamfard Co-authored-by: eoln <2881004+eoln@users.noreply.github.com> feat: add patch thirdparty request seed (#783) chore: add generate challenge endpoints (#790) * chore: add generate challenge endpoints * chore: fix description chore: add get transaction request seed (#791) --- .circleci/config.yml | 29 ++++++++---- package.json | 5 +- seeds/endpointType.js | 72 +++++++++++++++++++++++++++++ src/api/transactions/routes.js | 2 +- src/handlers/bulk/fulfil/handler.js | 9 ++++ 5 files changed, 106 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6864b8836..b1b26bb08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,6 +34,7 @@ defaults_awsCliDependencies: &defaults_awsCliDependencies | less \ mailcap pip3 install --upgrade pip awscli==1.14.5 s3cmd==2.0.1 python-magic + apk -v --purge del py-pip defaults_license_scanner: &defaults_license_scanner name: Install and set up license-scanner @@ -55,6 +56,7 @@ executors: default-machine: machine: image: ubuntu-1604:201903-01 + ## # Jobs # @@ -240,7 +242,7 @@ jobs: - run: name: Install docker dependencies for anchore command: | - apk add --update py-pip docker python3-dev libffi-dev openssl-dev gcc libc-dev make jq npm + apk add --update python3 py3-pip docker python3-dev libffi-dev openssl-dev gcc libc-dev make jq npm - run: name: Install general dependencies command: | @@ -316,12 +318,21 @@ jobs: command: | echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG" docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG - echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG" - docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG + case "$CIRCLE_TAG" in + *-pisp*) + # Don't update `late5t` for an image that has a `-pisp` + echo 'skipping late5t tag' + exit 0 + ;; + *) + echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG" + docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG + ;; + esac - slack/status: webhook: "$SLACK_WEBHOOK_ANNOUNCEMENT" success_message: '*"${CIRCLE_PROJECT_REPONAME}"* - Release \`"${CIRCLE_TAG}"\` \nhttps://github.com/mojaloop/"${CIRCLE_PROJECT_REPONAME}"/releases/tag/"${CIRCLE_TAG}"' - + deploy: executor: deploy-kube/helm-kube steps: @@ -411,7 +422,7 @@ workflows: - setup filters: tags: - only: /.*/ + only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?(\-pisp)?/ branches: ignore: - /feature*/ @@ -438,7 +449,7 @@ workflows: - audit-licenses filters: tags: - only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?/ + only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?(\-pisp)?/ branches: ignore: - /.*/ @@ -448,7 +459,7 @@ workflows: - build filters: tags: - only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?/ + only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?(\-pisp)?/ branches: ignore: - /.*/ @@ -458,7 +469,7 @@ workflows: - build filters: tags: - only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?/ + only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?(\-pisp)?/ branches: ignore: - /.*/ @@ -469,7 +480,7 @@ workflows: - image-scan filters: tags: - only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?/ + only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?(\-pisp)?/ branches: ignore: - /.*/ diff --git a/package.json b/package.json index 3587e9b85..86b1335a9 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "Sam Kummary ", "Steven Oderayi ", "Valentin Genev ", - "Shashikant Hirugade " + "Shashikant Hirugade ", + "Paweł Marzec ", + "Kevin Leylow " ], "repository": { "type": "git", @@ -79,6 +81,7 @@ "@hapi/good": "9.0.1", "@hapi/hapi": "20.0.2", "@hapi/inert": "6.0.3", + "@hapi/joi": "17.1.1", "@hapi/vision": "6.0.1", "@mojaloop/central-object-store": "11.0.1-snapshot", "@mojaloop/central-services-database": "10.6.1", diff --git a/seeds/endpointType.js b/seeds/endpointType.js index 98b4556b1..b212c51be 100644 --- a/seeds/endpointType.js +++ b/seeds/endpointType.js @@ -141,6 +141,78 @@ const endpointTypes = [ { name: 'FSPIOP_CALLBACK_URL_BULK_QUOTES', description: 'Bulk Quotes callback URL to which put bulkQuotes requests can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_GET', + description: 'Participant callback URL where GET /thirdpartyRequests/transactions/{ID} can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_POST', + description: 'Participant callback URL where POST /thirdpartyRequests/transactions can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_PUT', + description: 'Participant callback URL where PUT /thirdpartyRequests/transactions/{ID} can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_PUT_ERROR', + description: 'Participant callback URL to which PUT /thirdpartyRequests/transactions/{ID}/error error information can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_PATCH', + description: 'Participant callback URL where PATCH /thirdpartyRequests/transactions/{ID} can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_AUTH_POST', + description: 'Participant callback URL where POST /thirdpartyRequests/transactions/{ID}/authorizations can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_AUTH_PUT', + description: 'Participant callback URL where PUT /thirdpartyRequests/transactions/{ID}/authorizations can be sent' + }, + { + name: 'TP_CB_URL_TRANSACTION_REQUEST_AUTH_PUT_ERROR', + description: 'Participant callback URL where PUT /thirdpartyRequests/transactions/{ID}/authorizations/error error information can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_REQUEST_POST', + description: 'Participant callback URL where POST /consentRequests can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_REQUEST_PUT', + description: 'Participant callback URL where PUT /consentRequests/{ID} can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_REQUEST_PUT_ERROR', + description: 'Participant callback URL where PUT /consentRequests/{ID}/error error information can be sent' + }, + { + name: 'TP_CB_URL_CREATE_CREDENTIAL_POST', + description: 'Participant callback URL where POST /consentRequests/{ID}/createCredential can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_POST', + description: 'Participant callback URL where POST /consents/ can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_GET', + description: 'Participant callback URL where GET /consents/{ID} can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_PUT', + description: 'Participant callback URL where PUT /consents/{ID} can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_PUT_ERROR', + description: 'Participant callback URL where PUT /consents/{ID}/error error information can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_GENERATE_CHALLENGE_POST', + description: 'Participant callback URL where POST /consents/{ID}/generateChallenge can be sent' + }, + { + name: 'TP_CB_URL_CONSENT_GENERATE_CHALLENGE_PUT_ERROR', + description: 'Participant callback URL where PUT /consents/{ID}/generateChallenge/error error information can be sent' } ] diff --git a/src/api/transactions/routes.js b/src/api/transactions/routes.js index a26c01b67..da3b5889b 100644 --- a/src/api/transactions/routes.js +++ b/src/api/transactions/routes.js @@ -25,7 +25,7 @@ 'use strict' const Handler = require('./handler') -const Joi = require('joi') +const Joi = require('@hapi/joi') const tags = ['api', 'transaction'] module.exports = [ diff --git a/src/handlers/bulk/fulfil/handler.js b/src/handlers/bulk/fulfil/handler.js index 5da1b3593..2817e7dbd 100644 --- a/src/handlers/bulk/fulfil/handler.js +++ b/src/handlers/bulk/fulfil/handler.js @@ -149,6 +149,15 @@ const bulkFulfil = async (error, messages) => { while ((doc = await streamReader.readAsync()) !== null) { await sendIndividualTransfer(message, messageId, kafkaTopic, headers, payload, state, params, doc.payload, histTimerEnd) } + const dataUri = encodePayload(JSON.stringify(individualTransferFulfil), headers[Enum.Http.Headers.GENERAL.CONTENT_TYPE.value]) + const metadata = Util.StreamingProtocol.createMetadataWithCorrelatedEventState(message.value.metadata.event.id, Enum.Events.Event.Type.FULFIL, Enum.Events.Event.Action.COMMIT, Enum.Events.EventStatus.SUCCESS.status, Enum.Events.EventStatus.SUCCESS.code, Enum.Events.EventStatus.SUCCESS.description) // TODO: switch action to 'bulk-fulfil' flow + const msg = { + value: Util.StreamingProtocol.createMessage(messageId, headers[Enum.Http.Headers.FSPIOP.DESTINATION], headers[Enum.Http.Headers.FSPIOP.SOURCE], metadata, headers, dataUri, { id: transferId }) + } + params = { message: msg, kafkaTopic, consumer: Consumer, producer: Producer } + const eventDetail = { functionality: Enum.Events.Event.Type.FULFIL, action: Enum.Events.Event.Action.BULK_COMMIT } + await Kafka.proceed(Config.KAFKA_CONFIG, params, { consumerCommit, histTimerEnd, eventDetail }) + histTimerEnd({ success: true, fspId: Config.INSTRUMENTATION_METRICS_LABELS.fspId }) } } catch (err) { // TODO: handle individual transfers streaming error Logger.isInfoEnabled && Logger.info(Util.breadcrumb(location, `callbackErrorInternal2--${actionLetter}6`))