From bbae415b1c4c3018c7c05276913d8d5656f33edb Mon Sep 17 00:00:00 2001 From: Valentin Genev Date: Wed, 20 Nov 2019 14:36:38 +0200 Subject: [PATCH 1/6] added diagram of sending of trace to APM --- .../diagrams/sequence/process-flow.plantuml | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml diff --git a/mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml b/mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml new file mode 100644 index 000000000..e226dbc68 --- /dev/null +++ b/mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml @@ -0,0 +1,277 @@ +/'***** + License + -------------- + Copyright © 2017 Bill & Melinda Gates Foundation + The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + Contributors + -------------- + This is the official list of the Mojaloop project contributors for this file. + Names of the original copyright holders (individuals or organizations) + should be listed with a '*' in the first column. People who have + contributed from an organization can be listed under the organization + that actually holds the copyright for their contributions (see the + Gates Foundation organization for an example). Those individuals should have + their names indented and be marked with a '-'. Email address can be added + optionally within square brackets . + * Gates Foundation + - Name Surname + + * Valentin Genev + -------------- + ******'/ + +@startuml +' declate title +title Event Streaming Processor flow + +autonumber + +' Actor Keys: + +' declare actors + +collections "Notification topic" as TOPIC_NOTIFICATIONS +control "Topic Observable" as TOPIC_OBSERVABLE +control "Tracing Observable" as TRACING_OBSERVABLE +control "Caching Observable" as CACHING_OBSERVABLE +control "Check For Last Span Observable" as LASTSPAN_OBSERVABLE +control "Create Trace Observable" as CREATETRACE_OBSERVABLE +control "Cache Handler" as CACHE_HANDLER +control "Send Trace Handler" as TRACE_HANDLER +control "Send Span Handler" as SPAN_SENDER +database "Cache Storage" as CACHE +boundary "APM" as APM + +box "Central Services" #lightGray + participant TOPIC_NOTIFICATIONS +end box + +box "Event Stream Processor" #LightBlue + participant TOPIC_OBSERVABLE + participant TRACING_OBSERVABLE + participant CACHING_OBSERVABLE + participant LASTSPAN_OBSERVABLE + participant CREATETRACE_OBSERVABLE + participant TRACE_HANDLER + participant SPAN_SENDER + participant CACHE_HANDLER +end box + +box "Cache" #LightSteelBlue + participant CACHE +end box + +box "Kibana APM" #LightYellow + participant APM +end box + +' start flow + +group New Event Message Received + TOPIC_NOTIFICATIONS -> TOPIC_OBSERVABLE: Consume Event Message + activate TOPIC_OBSERVABLE + activate TOPIC_NOTIFICATIONS + deactivate TOPIC_NOTIFICATIONS + note over TOPIC_OBSERVABLE #yellow + Message: + { + "from": "payeefsp", + "to": "payerfsp", + "id": "659ee338-c8f8-4c06-8aff-944e6c5cd694", + "content": { + "headers": { + "content-type": "applicationvnd.interoperability.transfers+json;version=1.0", + "date": "2019-05-28T16:34:41.000Z", + "fspiop-source": "payeefsp", + "fspiop-destination": "payerfsp" + }, + "payload": + }, + "type": "application/json", + "metadata": { + "event": { + "id": "3920382d-f78c-4023-adf9-0d7a4a2a3a2f", + "type": "trace", + "action": "span", + "createdAt": "2019-05-29T23:18:32.935Z", + "state": { + "status": "success", + "code": 0, + "description": "action successful" + }, + "responseTo": "1a396c07-47ab-4d68-a7a0-7a1ea36f0012" + }, + "trace": { + "service": "central-ledger-prepare-handler", + "traceId": "bbd7b2c7bbd7b2c7", + "parentSpanId": "44ba9bbc5840", + "spanId": "2aa9cd0a7e87", + "startTimestamp": "2015-08-29T11:22:09.815479Z", + "finishTimestamp": "2015-08-29T11:22:09.815479Z", + "tags": { + "transctionId": "659ee338-c8f8-4c06-8aff-944e6c5cd694", + "transctionType": "transfer", + "parentEventType": "bulk-prepare", + "parentEventAction": "prepare" + } + } + } + } + end note + TOPIC_OBSERVABLE -> TOPIC_OBSERVABLE: Verify its tracing event Message + TOPIC_OBSERVABLE -> TRACING_OBSERVABLE: Send Message to Tracing Observable + deactivate TOPIC_OBSERVABLE + activate TRACING_OBSERVABLE + group Cache Span + TRACING_OBSERVABLE -> CACHING_OBSERVABLE: Send Span Context, \nmetadata.State and Content\nfrom Message + note right of TRACING_OBSERVABLE #yellow + { + spanContext: { + service: "central-ledger-prepare-handler", + traceId: "bbd7b2c7bbd7b2c7", + parentSpanId: "44ba9bbc5840", + spanId: "2aa9cd0a7e87", + startTimestamp: "2015-08-29T11:22:09.815479Z", + finishTimestamp: "2015-08-29T11:22:09.815479Z", + tags: { + transctionId: "659ee338-c8f8-4c06-8aff-944e6c5cd694", + transctionType: "transfer", + parentEventType: "bulk-prepare", + parentEventAction: "prepare" + }, + state: metadata.state, + content + } + end note + deactivate TRACING_OBSERVABLE + activate CACHING_OBSERVABLE + CACHING_OBSERVABLE <- CACHE: Get cachedTrace by traceId + alt the span should not be cached + CACHING_OBSERVABLE -> SPAN_SENDER: currentSpan + SPAN_SENDER -> APM: store to APM + CACHING_OBSERVABLE -> CACHING_OBSERVABLE: Complete + else + CACHING_OBSERVABLE <-> CACHING_OBSERVABLE: Validate transactionType, TransactionAction and service to match Config.START_CRITERIA && !parentSpandId + alt !cachedTrace + CACHING_OBSERVABLE -> CACHING_OBSERVABLE: Create new cachedTrace + note right of CACHING_OBSERVABLE #yellow + { + spans: {}, + masterSpan: null, + lastSpan: null + } + end note + end + alt !parentSpan + CACHING_OBSERVABLE <-> CACHING_OBSERVABLE: Generate MasterSpanId + CACHING_OBSERVABLE <-> CACHING_OBSERVABLE: Make received span child of masterSpan \nmerge({ parentSpanId: MasterSpanId }, { ...spanContext }) + CACHING_OBSERVABLE <-> CACHING_OBSERVABLE: Create MasterSpanContext merge({ tags: { ...tags, masterSpan: MasterSpanId } }, \n{ ...spanContext }, \n{ spanId: MasterSpanId, service: `master-${tags.transactionType}` }) + CACHING_OBSERVABLE <-> CACHING_OBSERVABLE: Add masterSpan to cachedTrace + end + CACHING_OBSERVABLE <-> CACHE_HANDLER: Add span to cachedTrace + note right of CACHING_OBSERVABLE #yellow + { + spans: { + 2aa9cd0a7e87: { + state, + content + spanContext: { + "service": "central-ledger-prepare-handler", + "traceId": "bbd7b2c7bbd7b2c7", + "parentSpanId": MasterSpanId, + "spanId": "2aa9cd0a7e87", + "startTimestamp": "2015-08-29T11:22:09.815479Z", + "finishTimestamp": "2015-08-29T11:22:09.815479Z", + "tags": { + "transctionId": "659ee338-c8f8-4c06-8aff-944e6c5cd694", + "transctionType": "transfer", + "parentEventType": "bulk-prepare", + "parentEventAction": "prepare" + } + } + }, + MasterSpanId: { + state, + content, + MasterSpanContext + } + }, + masterSpan: MasterSpanContext, + lastSpan: null + } + end note + CACHING_OBSERVABLE -> CACHE_HANDLER: Update cachedTrace to Cache + alt cachedTrace not staled or expired + CACHE_HANDLER -> CACHE_HANDLER: unsubscribe from Scheduler + CACHE_HANDLER -> CACHE: record cachedTrace + CACHE_HANDLER <-> CACHE_HANDLER: Reschedule scheduler for handling stale traces + end + end + CACHING_OBSERVABLE -> LASTSPAN_OBSERVABLE: Send traceId to check if last span has been received\nand cached + deactivate CACHING_OBSERVABLE + activate LASTSPAN_OBSERVABLE + end + group Check for last span + LASTSPAN_OBSERVABLE <- CACHE: Get cachedTrace by traceId + LASTSPAN_OBSERVABLE -> LASTSPAN_OBSERVABLE: Sort spans by startTimestamp + loop for currentSpan of SortedSpans + alt parentSpan + LASTSPAN_OBSERVABLE -> LASTSPAN_OBSERVABLE: isError = (errorCode in parentSpan \nOR parentSpan status === failed \nOR status === failed) + LASTSPAN_OBSERVABLE -> LASTSPAN_OBSERVABLE: apply masterSpan and error tags from parent to current span in cachedTrace + alt !isLastSpan + LASTSPAN_OBSERVABLE -> CACHE_HANDLER: Update cachedTrace to Cache + alt cachedTrace not staled or expired + CACHE_HANDLER -> CACHE_HANDLER: unsubscribe from Scheduler + CACHE_HANDLER -> CACHE: record cachedTrace + CACHE_HANDLER <-> CACHE_HANDLER: Reschedule scheduler for handling stale traces + end + else + LASTSPAN_OBSERVABLE -> LASTSPAN_OBSERVABLE: Validate transactionType, TransactionAction, service and isError \nto match Config.START_CRITERIA && !parentSpandId + LASTSPAN_OBSERVABLE -> LASTSPAN_OBSERVABLE: cachedTrace.lastSpan = currentSpan.spanContext + LASTSPAN_OBSERVABLE -> CACHE_HANDLER: Update cachedTrace to Cache + alt cachedTrace not staled or expired + CACHE_HANDLER -> CACHE_HANDLER: unsubscribe from Scheduler + CACHE_HANDLER -> CACHE: record cachedTrace + CACHE_HANDLER <-> CACHE_HANDLER: Reschedule scheduler for handling stale traces + end + LASTSPAN_OBSERVABLE -> CREATETRACE_OBSERVABLE: Send traceId + deactivate LASTSPAN_OBSERVABLE + activate CREATETRACE_OBSERVABLE + end + end + end + end + group Recreate Trace from Cached Trace + CREATETRACE_OBSERVABLE <- CACHE: get cachedTrace by TraceId + alt cachedTrace.lastSpan AND cachedTrace.masterSpan + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: currentSpan = lastSpan + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: resultTrace = [lastSpan] + loop for i = 0; i < cachedTrace.spans.length; i++ + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: get parentSpan of currentSpan + alt parentSpan + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: insert parent span in resultTrace in front + else + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: break loop + end + end + alt cachedTrace.masterSpan === currentSpan.spanId + CREATETRACE_OBSERVABLE -> CREATETRACE_OBSERVABLE: masterSpan.finishTimestamp = resultTrace[resultTrace.length - 1].finishTimestamp + CREATETRACE_OBSERVABLE -> TRACE_HANDLER: send resultTrace + deactivate CREATETRACE_OBSERVABLE + activate TRACE_HANDLER + end + end + end + group send Trace + loop trace elements + TRACE_HANDLER -> SPAN_SENDER: send each span + SPAN_SENDER -> APM: send span to APM + end + TRACE_HANDLER -> TRACE_HANDLER: unsubscribe scheduler for traceId + TRACE_HANDLER -> CACHE: drop cachedTrace + + end + end +@enduml \ No newline at end of file From 9ab693c17b9e39d6c3d10d3935b45faa24a7f747 Mon Sep 17 00:00:00 2001 From: Valentin Date: Mon, 16 Dec 2019 11:07:40 +0200 Subject: [PATCH 2/6] added documentation for event stream processor and updated the Event Framework tags information --- SUMMARY.md | 1 + .../event-framework/README.md | 1 + .../event-stream-processor/README.md | 13 ++++ .../event-stream-processor-overview.svg | 1 + .../diagrams/sequence/process-flow.plantuml | 0 package-lock.json | 65 ++++++++++--------- package.json | 1 + 7 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 mojaloop-technical-overview/event-stream-processor/README.md create mode 100644 mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg rename mojaloop-technical-overview/{event-streaming-processor => event-stream-processor}/assets/diagrams/sequence/process-flow.plantuml (100%) diff --git a/SUMMARY.md b/SUMMARY.md index ee2f2772c..5604a65be 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -110,6 +110,7 @@ * [Notification Handler For Rejections](mojaloop-technical-overview/central-event-processor/5.1.1-notification-handler-for-rejections.md) * [Signature Validation](mojaloop-technical-overview/central-event-processor/signature-validation.md) * [Event Framework](mojaloop-technical-overview/event-framework/README.md) + * [Event Stream Processor](mojaloop-technical-overview/event-stream-processor/README.md) * [Fraud Services](mojaloop-technical-overview/fraud-services/README.md) * [Ecosystem Fraud Documentation](mojaloop-technical-overview/fraud-services/related-documents/documentation.md) * [SDK Scheme Adapter](mojaloop-technical-overview/sdk-scheme-adapter/README.md) diff --git a/mojaloop-technical-overview/event-framework/README.md b/mojaloop-technical-overview/event-framework/README.md index 18acbf248..a8a486c06 100644 --- a/mojaloop-technical-overview/event-framework/README.md +++ b/mojaloop-technical-overview/event-framework/README.md @@ -143,6 +143,7 @@ _Note: HEXDIGLC = DIGIT / "a" / "b" / "c" / "d" / "e" / "f" ; lower case hex cha | transactionType | string | N | The transaction type represented by the transactionId. E.g. (transfer, quote, etc) | transfer | | parentEventType | string | N | The event-type of the parent Span. | bulk-prepare | | parentEventAction | string | N | The event-action of the parent Span. | prepare | +| tracestate | string | N | This tag is set if EventSDK environmental variable `EVENT_SDK_TRACESTATE_HEADER_ENABLED` is `true` or if parent span context has the `tracestate` header or tag included. The tag holds updated `tracestate` header value as per the W3C trace headers specification. [More here](#411-wc3-http-headers)| `congo=t61rcWkgMzE,rojo=00f067aa0ba902b7` | | `` | string | N | Arbitary Key-value pair for additional meta-data to be added to the trace information. | n/a | ##### 3.2.1.6 Enum: EventStatusType diff --git a/mojaloop-technical-overview/event-stream-processor/README.md b/mojaloop-technical-overview/event-stream-processor/README.md new file mode 100644 index 000000000..f51c184f4 --- /dev/null +++ b/mojaloop-technical-overview/event-stream-processor/README.md @@ -0,0 +1,13 @@ +# Event Stream Processor Service + +Event Stream Processor consumes event messages from all services side cars as they are created according to the [Event Framework](../event-framework/README.md). + +The Service delivers logs, including audits, and traces to EFK stack with enabled APM plugin. Based on the event message type, the messages are delivered to different indexes in the Elasticsearch. + + +![Event Stream Processor flow overview](./assets/diagrams/architecture/event-stream-processor-overview.svg) + +## Trace processing flow sequence diagram +{% uml src="./mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml" %} +{% enduml %} + diff --git a/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg new file mode 100644 index 000000000..310537d30 --- /dev/null +++ b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml similarity index 100% rename from mojaloop-technical-overview/event-streaming-processor/assets/diagrams/sequence/process-flow.plantuml rename to mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml diff --git a/package-lock.json b/package-lock.json index b53b5818c..443036250 100644 --- a/package-lock.json +++ b/package-lock.json @@ -576,7 +576,7 @@ "colors": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + "integrity": "sha1-xQSRR51MG9rtLJztMs98fcI2D3g=" }, "combined-stream": { "version": "1.0.8", @@ -698,7 +698,7 @@ "cssom": { "version": "0.3.8", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "integrity": "sha1-nxJ29bK0Y/IRTT8sdSUK+MGjb0o=", "optional": true }, "cssstyle": { @@ -1003,7 +1003,7 @@ "escodegen": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.12.0.tgz", - "integrity": "sha512-TuA+EhsanGcme5T3R0L80u4t8CpbXQjegRmf7+FPTJrtCTErXFeelblRgHQa1FofEzqYYJmJ/OqjTwREp9qgmg==", + "integrity": "sha1-92Pa+ECvFyuzorbdchnA4X9/9UE=", "optional": true, "requires": { "esprima": "^3.1.3", @@ -1022,7 +1022,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", "optional": true } } @@ -1035,13 +1035,13 @@ "estraverse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "integrity": "sha1-OYrT88WiSUi+dyXoPRGn3ijNvR0=", "optional": true }, "esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "integrity": "sha1-dNLrTeC42hKTcRkQ1Qd1ubcQ72Q=", "optional": true }, "etag": { @@ -2131,7 +2131,7 @@ "gitbook-plugin-changelog": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gitbook-plugin-changelog/-/gitbook-plugin-changelog-1.0.1.tgz", - "integrity": "sha512-X1wDTUfBRLiuS29SBN6e4X2A08JsGMh/KsAqBblrPQ2ltbM2Ivb6i2k9FinH4AVCdvdXEuM+KgQI8Q2tQzujtQ==", + "integrity": "sha1-CaEBwqXHm7q1NDyI3/qvZ9GPCVE=", "requires": { "lodash": "^4.17.4", "moment": "^2.18.1" @@ -2147,6 +2147,11 @@ "resolved": "https://registry.npmjs.org/gitbook-plugin-editlink/-/gitbook-plugin-editlink-1.0.2.tgz", "integrity": "sha1-ej2Bk/guYfCUF83w52h2MfW2GV4=" }, + "gitbook-plugin-fontsettings": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/gitbook-plugin-fontsettings/-/gitbook-plugin-fontsettings-2.0.0.tgz", + "integrity": "sha1-g1+QCuPdERCG/n7UQl7j3gJIYas=" + }, "gitbook-plugin-include": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/gitbook-plugin-include/-/gitbook-plugin-include-0.1.0.tgz", @@ -2163,12 +2168,12 @@ "gitbook-plugin-page-toc": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/gitbook-plugin-page-toc/-/gitbook-plugin-page-toc-1.1.1.tgz", - "integrity": "sha512-0ii4d4gxaLsZuWvSFyz2eLQvpjB3Sjg8pTBW8P6mT0j1jw3ifS5Jx8HvQLYrOPyolktUqmd0AebrKCur667zfw==" + "integrity": "sha1-YEZUSvoRNEJ2nwwLO0F8H+TfQRs=" }, "gitbook-plugin-plantuml-svg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gitbook-plugin-plantuml-svg/-/gitbook-plugin-plantuml-svg-1.0.1.tgz", - "integrity": "sha512-YP3nmYI+6j2aUPk3hKkWkggIlORgGwLhvaoReZvdyVM5cvomyd9iq4leRMvxBUSwuD6I1fXyYLMGqunOIjkXbw==", + "integrity": "sha1-4QfoSCp/Gu3j0/IXzk1C/BWe6Qk=", "requires": { "js-base64": "^2.4.0", "lodash": "^4.17.4", @@ -2316,7 +2321,7 @@ "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "integrity": "sha1-tEf2ZwoEVbv+7dETku/zMOoJdUg=" } } }, @@ -2363,7 +2368,7 @@ "handlebars": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-3.0.7.tgz", - "integrity": "sha512-Pb/VCTFwbcm3KbD5rIzVVsbVBk7XMfhS4ZIJUfXdMF7H+UngtsofnUBfop/i24GSr3HCDG1PL0KoIX0YqEsXTg==", + "integrity": "sha1-1k8NakgOCmN0IK91cTR1FmotZEs=", "requires": { "optimist": "^0.6.1", "source-map": "^0.1.40", @@ -2751,12 +2756,12 @@ "js-base64": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.5.1.tgz", - "integrity": "sha512-M7kLczedRMYX4L8Mdh4MzyAMM9O5osx+4FcOQuTvr3A9F2D9S5JXheN0ewNbrvK2UatkTRhL5ejGmGSjNMiZuw==" + "integrity": "sha1-Hvo57yxfeYC7F4St5KivLeMpESE=" }, "js-quantities": { "version": "1.7.4", "resolved": "https://registry.npmjs.org/js-quantities/-/js-quantities-1.7.4.tgz", - "integrity": "sha512-mjwSQ+1XwSTqStOQNocgRgwebWZYc4CDhoe9DFY51u0n23B/VQmwCb4z1GTn9vxz/gMXHTnnyCdXBGrCMbMy8g==" + "integrity": "sha1-JPlgZr8aV6Uh/RCesdbgFYzNddc=" }, "js-yaml": { "version": "3.13.1", @@ -2803,7 +2808,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=" }, "json-stable-stringify": { "version": "1.0.1", @@ -3128,7 +3133,7 @@ "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", - "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "integrity": "sha1-gEs6e2qZNYw8XGHnHYco8EHP+Rc=", "requires": { "anymatch": "^2.0.0", "async-each": "^1.0.1", @@ -3621,7 +3626,7 @@ "merge2": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.3.0.tgz", - "integrity": "sha512-2j4DAdlBOkiSZIsaXk4mTE3sRS02yBHAtfy127xRV3bQUFqXkjHCHLW6Scv7DwNRbIWNHH8zpnz9zMaKXIdvYw==" + "integrity": "sha1-WzZu6DsvFYLEj4fkfPGpNSEDyoE=" }, "methods": { "version": "1.1.2", @@ -3682,7 +3687,7 @@ "mixin-deep": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", - "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "integrity": "sha1-ESC0PcNZp4Xc5ltVuC4lfM9HlWY=", "requires": { "for-in": "^1.0.2", "is-extendable": "^1.0.1" @@ -3691,7 +3696,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", "requires": { "is-plain-object": "^2.0.4" } @@ -3716,7 +3721,7 @@ "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + "integrity": "sha1-DQVdU/UFKqZTyfbraLtdEr9cK1s=" }, "morgan": { "version": "1.9.1", @@ -8123,7 +8128,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha1-eCDZsWEgzFXKmud5JoCufbptf+I=" }, "progress": { "version": "1.1.8", @@ -8638,7 +8643,7 @@ "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "integrity": "sha1-NXl/E6f9rcVmFCwp1PB8ytSD4+w=", "requires": { "glob": "^7.1.3" } @@ -8745,7 +8750,7 @@ "set-value": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", - "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "integrity": "sha1-oY1AUw5vB95CKMfe/kInr4ytAFs=", "requires": { "extend-shallow": "^2.0.1", "is-extendable": "^0.1.1", @@ -9018,7 +9023,7 @@ "symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "integrity": "sha1-QwY30ki6d+B4iDlR+5qg7tfGP6I=", "optional": true }, "throttleit": { @@ -9080,7 +9085,7 @@ "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + "integrity": "sha1-fhvjRw8ed5SLxD2Uo8j013UrpVM=" }, "tough-cookie": { "version": "2.4.3", @@ -9184,7 +9189,7 @@ "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", - "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "integrity": "sha1-C2/nuDWuzaYcbqTU8CwUIh4QmEc=", "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", @@ -9195,7 +9200,7 @@ "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + "integrity": "sha1-tkb2m+OULavOzJ1mOcgNwQXvqmY=" }, "unix-crypt-td-js": { "version": "1.0.0", @@ -9251,12 +9256,12 @@ "upath": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", - "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==" + "integrity": "sha1-j2bbzVWog6za5ECK+LA1pQRMGJQ=" }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", "requires": { "punycode": "^2.1.0" } @@ -9292,7 +9297,7 @@ "uuid": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", - "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + "integrity": "sha1-RWjwIW54dg7h2/Ok0s9T4iQRKGY=" }, "vary": { "version": "1.1.2", @@ -9318,7 +9323,7 @@ "websocket-driver": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.3.tgz", - "integrity": "sha512-bpxWlvbbB459Mlipc5GBzzZwhoZgGEZLuqPaR0INBGnPAY1vdBX6hPnoFXiw+3yWxDuHyQjO2oXTMyS8A5haFg==", + "integrity": "sha1-otTg1PTxFvHmKX66WLBdQwEA6fk=", "requires": { "http-parser-js": ">=0.4.0 <0.4.11", "safe-buffer": ">=5.1.0", diff --git a/package.json b/package.json index 8c071a0c3..6ec14e7b3 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "gitbook-plugin-changelog": "1.0.1", "gitbook-plugin-collapsible-chapters": "0.1.8", "gitbook-plugin-editlink": "1.0.2", + "gitbook-plugin-fontsettings": "^2.0.0", "gitbook-plugin-include": "0.1.0", "gitbook-plugin-insert-logo": "0.1.5", "gitbook-plugin-page-toc": "1.1.1", From 01e5cfb761b9d51304956a4ef591a46e1d2c2216 Mon Sep 17 00:00:00 2001 From: Valentin Date: Wed, 18 Dec 2019 15:35:48 +0200 Subject: [PATCH 3/6] added section for custom mojaloop schema, updated diagrams --- .../event-stream-processor/README.md | 31 +++++++++++++++++-- .../event-stream-processor-overview.svg | 4 ++- .../diagrams/sequence/process-flow.plantuml | 5 ++- package.json | 2 +- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/mojaloop-technical-overview/event-stream-processor/README.md b/mojaloop-technical-overview/event-stream-processor/README.md index f51c184f4..aae1421e9 100644 --- a/mojaloop-technical-overview/event-stream-processor/README.md +++ b/mojaloop-technical-overview/event-stream-processor/README.md @@ -1,13 +1,40 @@ # Event Stream Processor Service -Event Stream Processor consumes event messages from all services side cars as they are created according to the [Event Framework](../event-framework/README.md). +Event Stream Processor consumes event messages from the `topic-events` topic as a result of messages published by the [event-sidecar](https://github.com/mojaloop/event-sidecar) service. Refer to [Event Framework](../event-framework/README.md) for more information on the overall architecture. The Service delivers logs, including audits, and traces to EFK stack with enabled APM plugin. Based on the event message type, the messages are delivered to different indexes in the Elasticsearch. +## 1. Pre-requisites + +### 1.1 Elasticsearch + +Ensure that you have created the `mojatemplate` based on the following config: [template](https://github.com/mojaloop/event-stream-processor/blob/master/config/template-mojaloop.json). + +#### 1.1.1 Create Template + ```curl + curl -X PUT "http://elasticsearch:9200/_template/mojatemplate?pretty" -H 'Content-Type: application/json' -d @config/template-mojaloop.json' + ``` + +#### 1.1.2 Delete Template +_Note: only needed if you need to remove the template_ + ```curl + curl -X DELETE "http://elasticsearch:9200/_template/mojatemplate" + ``` + + #### 1.1.3 Get Template + _Note: useful for debugging template issues_ + ```curl + curl -X GET "http://elasticsearch:9200/_template/mojatemplate" + ``` + +## 2. Architecture overview + +### 2.1. Flow overview ![Event Stream Processor flow overview](./assets/diagrams/architecture/event-stream-processor-overview.svg) -## Trace processing flow sequence diagram + +### 2.2 Trace processing flow sequence diagram {% uml src="./mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml" %} {% enduml %} diff --git a/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg index 310537d30..9deb189e9 100644 --- a/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg +++ b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/architecture/event-stream-processor-overview.svg @@ -1 +1,3 @@ - \ No newline at end of file + + +
topic-event
topic-eventEvent Stream Processor
Send to Elasticsearch mojaloop custom schema via Elasticsearch API
[Not supported by viewer]
Yes
Yes
Is it trace?
Is it trace?
No
No
Should create master span?
Should create master span?
exit
exit
No
No
APM Agent
APM Agent
Create cache for the trace
Create cache for the trace
Yes
Yes
No
No
Is last span?
Is last span?
Recreate trace
Recreate trace
exit
exit
EFK Stack (Elasticsearch, Kibana, APM)
EFK Stack (Elasticsearch, Kibana, APM) \ No newline at end of file diff --git a/mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml index e226dbc68..9817a79d8 100644 --- a/mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml +++ b/mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml @@ -43,6 +43,7 @@ control "Send Trace Handler" as TRACE_HANDLER control "Send Span Handler" as SPAN_SENDER database "Cache Storage" as CACHE boundary "APM" as APM +boundary "Elasticsearch API" as ELASTIC box "Central Services" #lightGray participant TOPIC_NOTIFICATIONS @@ -63,8 +64,9 @@ box "Cache" #LightSteelBlue participant CACHE end box -box "Kibana APM" #LightYellow +box "Elasticsearch" #LightYellow participant APM + participant ELASTIC end box ' start flow @@ -120,6 +122,7 @@ group New Event Message Received } } end note + TOPIC_OBSERVABLE -> ELASTIC: Send message for log purposes to custom index TOPIC_OBSERVABLE -> TOPIC_OBSERVABLE: Verify its tracing event Message TOPIC_OBSERVABLE -> TRACING_OBSERVABLE: Send Message to Tracing Observable deactivate TOPIC_OBSERVABLE diff --git a/package.json b/package.json index 6ec14e7b3..ef80e2380 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "gitbook-plugin-changelog": "1.0.1", "gitbook-plugin-collapsible-chapters": "0.1.8", "gitbook-plugin-editlink": "1.0.2", - "gitbook-plugin-fontsettings": "^2.0.0", + "gitbook-plugin-fontsettings": "2.0.0", "gitbook-plugin-include": "0.1.0", "gitbook-plugin-insert-logo": "0.1.5", "gitbook-plugin-page-toc": "1.1.1", From 8fff89be48dad8ae18a3dffcd273a703a3af7801 Mon Sep 17 00:00:00 2001 From: Valentin Date: Fri, 3 Jan 2020 13:21:17 +0200 Subject: [PATCH 4/6] edit the custom template section --- .../event-stream-processor/README.md | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/mojaloop-technical-overview/event-stream-processor/README.md b/mojaloop-technical-overview/event-stream-processor/README.md index aae1421e9..427b3246d 100644 --- a/mojaloop-technical-overview/event-stream-processor/README.md +++ b/mojaloop-technical-overview/event-stream-processor/README.md @@ -4,37 +4,17 @@ Event Stream Processor consumes event messages from the `topic-events` topic as The Service delivers logs, including audits, and traces to EFK stack with enabled APM plugin. Based on the event message type, the messages are delivered to different indexes in the Elasticsearch. +## 1. Prerequisites + +The service logs all the events to Elasticsearch instance with enabled APM plugin. Sample docker-compose of the Elastic stack is available [here](https://github.com/mojaloop/event-stream-processor/blob/master/test/util/scripts/docker-efk/docker-compose.yml). The logs and audits are created under custom index template, while the trace data is stored to the default `apm-*` index. +Please, ensure that you have created the `mojatemplate` as it is described into the [event-stream-processor](https://github.com/mojaloop/event-stream-processor#111-create-template) service documentation. -## 1. Pre-requisites - -### 1.1 Elasticsearch - -Ensure that you have created the `mojatemplate` based on the following config: [template](https://github.com/mojaloop/event-stream-processor/blob/master/config/template-mojaloop.json). - -#### 1.1.1 Create Template - ```curl - curl -X PUT "http://elasticsearch:9200/_template/mojatemplate?pretty" -H 'Content-Type: application/json' -d @config/template-mojaloop.json' - ``` - -#### 1.1.2 Delete Template -_Note: only needed if you need to remove the template_ - ```curl - curl -X DELETE "http://elasticsearch:9200/_template/mojatemplate" - ``` - - #### 1.1.3 Get Template - _Note: useful for debugging template issues_ - ```curl - curl -X GET "http://elasticsearch:9200/_template/mojatemplate" - ``` - ## 2. Architecture overview ### 2.1. Flow overview -![Event Stream Processor flow overview](./assets/diagrams/architecture/event-stream-processor-overview.svg) +![Event Stream Processor flow overview](./assets/diagrams/architePcture/event-stream-processor-overview.svg) ### 2.2 Trace processing flow sequence diagram {% uml src="./mojaloop-technical-overview/event-stream-processor/assets/diagrams/sequence/process-flow.plantuml" %} {% enduml %} - From 9eeb8018177f2ad0e9bf68db394612ae927fa024 Mon Sep 17 00:00:00 2001 From: Miguel de Barros Date: Fri, 24 Jan 2020 19:01:27 +0200 Subject: [PATCH 5/6] Updated component arch diagram for PI8, and also added legacy links --- SUMMARY.md | 3 +++ .../diagrams/architecture/architecture-event-trace.svg | 2 +- .../diagrams/architecture/Arch-Mojaloop-overview-PI8.svg | 3 +++ mojaloop-technical-overview/overview/components-PI6.md | 7 +++++++ mojaloop-technical-overview/overview/components-PI7.md | 7 +++++++ mojaloop-technical-overview/overview/components-PI8.md | 7 +++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 mojaloop-technical-overview/overview/assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg create mode 100644 mojaloop-technical-overview/overview/components-PI6.md create mode 100644 mojaloop-technical-overview/overview/components-PI7.md create mode 100644 mojaloop-technical-overview/overview/components-PI8.md diff --git a/SUMMARY.md b/SUMMARY.md index 5604a65be..70f4ecb62 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -25,6 +25,9 @@ * [Documentation Style Guide](contributors-guide/documentation/documentation-style-guide.md) * [Mojaloop Technical Overview](mojaloop-technical-overview/README.md) * [Mojaloop Hub](mojaloop-technical-overview/overview/README.md) + * [Current Architecture - PI8](mojaloop-technical-overview/overview/components-PI8.md) + * [Legacy Architecture - PI7](mojaloop-technical-overview/overview/components-PI7.md) + * [Legacy Architecture - PI6](mojaloop-technical-overview/overview/components-PI6.md) * [Legacy Architecture - PI5](mojaloop-technical-overview/overview/components-PI5.md) * [Legacy Architecture - PI3](mojaloop-technical-overview/overview/components-PI3.md) * [Account-Lookup Service](mojaloop-technical-overview/account-lookup-service/README.md) diff --git a/mojaloop-technical-overview/event-framework/assets/diagrams/architecture/architecture-event-trace.svg b/mojaloop-technical-overview/event-framework/assets/diagrams/architecture/architecture-event-trace.svg index 1a6436629..5fd81ccfb 100644 --- a/mojaloop-technical-overview/event-framework/assets/diagrams/architecture/architecture-event-trace.svg +++ b/mojaloop-technical-overview/event-framework/assets/diagrams/architecture/architecture-event-trace.svg @@ -1,3 +1,3 @@ -
Sidecar
Sidecar
Sidecar
Sidecar
Service
Consumer 
[Not supported by viewer]
Parent Span
Parent Span
Trace
Trace
Start
Start<br>
End
End<br>
Child Span
Child Span
Start
Start<br>
End
End<br>
Child Span
Child Span
Start
Start<br>
End
End<br>
Trace Context
- traceId
- spanId
- parentId
- sampled
- flags
[Not supported by viewer]
Service
Producer 
[Not supported by viewer]
Tags: [ transactionId=transferId, source=payerFsp, 
destination=payeeFsp ]
[Not supported by viewer]
Child Span
Child Span
Start
Start<br>
End
End<br>
Message Transport
- HTTP
- Kafka
[Not supported by viewer]
Error
Error<br>
<div style="text-align: center"></div>
Kafka
(Event Topic)
[Not supported by viewer]
Event Sidecar
Event Sidecar
Event Sidecar
Event Sidecar
Record Trace
Record Trace<br>
Service A
Service A
Service B
Service B
Trace Message
- traceId
- spanId
- parentId
- sampled
- flags
- tags
- content / error
[Not supported by viewer]
Record Trace
Record Trace<br>
Record Trace
Record Trace<br>
Record Trace
Record Trace<br>
Record Error
Record Error<br>
Audit
Audit<br>
Record
Audit
[Not supported by viewer]
Log
Log<br>
Record Log
Record Log<br> \ No newline at end of file +
Sidecar
Sidecar
Sidecar
Sidecar
Service
Consumer 
Service...
Parent Span
Parent Span
Trace
Trace
Start
Start
End
End
Child Span
Child Span
Start
Start
End
End
Child Span
Child Span
Start
Start
End
End
Trace Context
- traceId
- spanId
- parentId
- sampled
- flags
- tracestate
Trace Context...
Service
Producer 
Service...
Tags: [ transactionId: transferId, transactionType: transfer, source: payerFsp, destination: payeeFsp, tracestate: mojaloop=f32c19568004ce8b ]
Tags: [ transactionId: transferId, transactionType: transfer, source: payerFsp, destination: payeeFsp, tracestate: mojaloop=f32c19568004ce...
Child Span
Child Span
Start
Start
End
End
Message Transport
- HTTP
- Kafka
Message Transport...
Error
Error
Kafka
(Event Topic)
Kafka...
Event Sidecar
Event Sidecar
Event Sidecar
Event Sidecar
Record Trace
Record...
Service A
Service A
Service B
Service B
Trace Message
- traceId
- spanId
- parentId
- sampled
- flags
- tags
- tracestate
- content / error
Trace Message...
Record Trace
Record...
Record Trace
Record...
Record Trace
Record...
Record Error
Record...
Audit
Audit
Record
Audit
Record...
Log
Log
Record Log
Record...
Record
Audit
Record...
Record
Audit
Record...
Record
Audit
Record...
Record
Audit
Record... \ No newline at end of file diff --git a/mojaloop-technical-overview/overview/assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg b/mojaloop-technical-overview/overview/assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg new file mode 100644 index 000000000..3e5b75f13 --- /dev/null +++ b/mojaloop-technical-overview/overview/assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg @@ -0,0 +1,3 @@ + + +
Operational Monitoring
Operational Monitoring
Quality Assurance
Quality Assurance
Core Components
Core Components
API - Security / Policy / Ingress / Egress
API - Security / Policy / Ingress / Egress
Continuous Integration & Delivery
Continuous Integration & Delivery
Container Management & Orchestration
Container Management & Orchestration
Supporting Components
Supporting Components
Helm
(Package Manager for K8s, Templatized Deployments and Configuration)
Helm...
Kubernetes
(Abstraction layer for Infrastructure, Infrastructure as a Policy, Container Orchestration)
Kubernetes...
Docker
(Container Engine)
Docker...
Infrastructure
(AWS, Azure, On-Prem)
Infrastructure...
EFK - ElasticSearch / Fluentd / Kibana
(Log Search / Collection / Monitoring / Tracing)
EFK - ElasticSearch / Fluentd / Kibana...
Promfana - Prometheus / Grafana
(Log Search / Collection / Monitoring)
Promfana - Prometheus / Grafana...
Open Tracing
(Distributed Tracing)
Open Tracing...
Rancher
(Infrastructure Management)
Rancher...
Mojaloop API Adapter
(API - Transfers)
Mojaloop API Adapter...
Central-Services
(API - Operational Admin)
Central-Services...
Central-Services
(Handler - Prepare)
Central-Services...
Central-Services
(Handler - Position)
Central-Services...
Central-Services
(Handler - Fulfil)
Central-Services...
Central-Services
(Handler - Get Transfers)
Central-Services...
Central-Services
(Handler - Timeout)
Central-Services...
Central-KMS
(Future - Key Management Store)
Central-KMS...
ALS - Account-Lookup-Service
(API - Parties, Participant)
ALS - Account-Lookup-Service...
ALS-Oracle-Pathfinder
(MSISDN Lookup)
ALS-Oracle-Pathfinder...
Central-Event-Processor
(CEP)
Central-Event-Processor...
Email-Notifier
(Handler - Email)
Email-Notifier...
Central-Services
(Handler - Admin)
Central-Services...
Circle-CI
(Test, Build, Deploy)
Circle-CI...
Docker Hub
(Container Repository)
Docker Hub...
NPM Org
(NPM Repository)
NPM Org...
GitBooks
(Documetation)
GitBooks...
API Gateway
(Future - API - Parties, Participants, Quotes, Transfers, Bulk Transfers)
API Gateway...
Central-Services
(Handler - Bulk Prepare)
Central-Services...
Central-Services
(Handler - Bulk Fulfil)
Central-Services...
Central-Services
(Handler - Bulk Process)
Central-Services...
Mojaloop API Adapter
(API - Bulk Transfers)
Mojaloop API Adapter...
Forensic-Logging
(Future - Auditing)
Forensic-Logging...
Central-Settlements
(API - Settlement)
Central-Settlements...
Event-Stream-Processor
Logs, Error, Audits, Tracing)
Event-Stream-Processor...
Simulators
(API, SDK, FSP & Oracle)
Simulators...
On-boarding
(Postman - Scripts)
On-boarding...
Functional Tests
(Postman - Scripts)
Functional Tests...
Self Testing Toolkit
(POC - UI)
Self Testing Toolkit...
License, Security & Audit
(Dependencies)
License, Security & Audit...
Persistence
Persistence
Redis
(SDK - Cache)
Redis...
MongoDB
(CEP - Data)
MongoDB...
MySQL
(Percona - Data)
MySQL...
Kafka
(Message - Streaming)
Kafka...
Transaction Request Service
(API - Transaction)
Transaction Request Service...
Software Dev Kits
Software Dev Kits
Oracle SDK
(Participant Store)
Oracle SDK...
Mojaloop SDK 
(FSP Payer/Payee)
Mojaloop SDK...
Event SDK
(Audit, Log, Trace - Client/Server)
Event SDK...
Event-Sidecar
(Logs, Error, Audits, Tracing)
Event-Sidecar...
Mojaloop API Adapter
(Handler - Notifications)
Mojaloop API Adapter...
Quoting-Service
(API - Quotes)
Quoting-Service...
Finance-Portal-UI
(API - Bulk Transfers)
Finance-Portal-UI...
Settlement-Management
(Closing, Recon report, Cronjob)
Settlement-Management...
Schema-Adapter
(SDK - Parties, Participants, Quotes, Transfers)
Schema-Adapter...
IaC (Infra as Code) Tools
(Future - Automated
Prov / Depl / Upgrades)
IaC (Infra as Code) Tools... \ No newline at end of file diff --git a/mojaloop-technical-overview/overview/components-PI6.md b/mojaloop-technical-overview/overview/components-PI6.md new file mode 100644 index 000000000..a935f7cc5 --- /dev/null +++ b/mojaloop-technical-overview/overview/components-PI6.md @@ -0,0 +1,7 @@ +# Mojaloop Hub Legacy Components - PI6 + +The following component diagram shows the break-down of the Mojaloop services and its micro-service architecture for pre PI6: + +![Mojaloop Architecture Overview PI6](./assets/diagrams/architecture/Arch-Mojaloop-overview-PI6.svg) + +_Note: Colour-grading indicates the relationship between data-store, and message-streaming / adapter-interconnects. E.g. `Central-Services` utilise `MySQL` as a Data-store, and leverage on `Kafka` for Messaging_ diff --git a/mojaloop-technical-overview/overview/components-PI7.md b/mojaloop-technical-overview/overview/components-PI7.md new file mode 100644 index 000000000..f2f59dd8a --- /dev/null +++ b/mojaloop-technical-overview/overview/components-PI7.md @@ -0,0 +1,7 @@ +# Mojaloop Hub Legacy Components - PI7 + +The following component diagram shows the break-down of the Mojaloop services and its micro-service architecture for pre PI7: + +![Mojaloop Architecture Overview PI7](./assets/diagrams/architecture/Arch-Mojaloop-overview-PI7.svg) + +_Note: Colour-grading indicates the relationship between data-store, and message-streaming / adapter-interconnects. E.g. `Central-Services` utilise `MySQL` as a Data-store, and leverage on `Kafka` for Messaging_ diff --git a/mojaloop-technical-overview/overview/components-PI8.md b/mojaloop-technical-overview/overview/components-PI8.md new file mode 100644 index 000000000..383a75614 --- /dev/null +++ b/mojaloop-technical-overview/overview/components-PI8.md @@ -0,0 +1,7 @@ +# Mojaloop Hub Legacy Components - PI8 + +The following component diagram shows the break-down of the Mojaloop services and its micro-service architecture for pre PI8: + +![Mojaloop Architecture Overview PI8](./assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg) + +_Note: Colour-grading indicates the relationship between data-store, and message-streaming / adapter-interconnects. E.g. `Central-Services` utilise `MySQL` as a Data-store, and leverage on `Kafka` for Messaging_ From 859a846024ad01613550ef9e084b6940b7751d7a Mon Sep 17 00:00:00 2001 From: Miguel de Barros Date: Fri, 31 Jan 2020 17:07:52 +0200 Subject: [PATCH 6/6] Updated overview to point to latest component arch overview diagram --- mojaloop-technical-overview/overview/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mojaloop-technical-overview/overview/README.md b/mojaloop-technical-overview/overview/README.md index 9d0cb803c..78e405de9 100644 --- a/mojaloop-technical-overview/overview/README.md +++ b/mojaloop-technical-overview/overview/README.md @@ -4,7 +4,7 @@ There are several components that make up the Mojaloop ecosystem. The Mojaloop H The following component diagram shows the break-down of the Mojaloop services and its micro-service architecture: -![Current Mojaloop Architecture Overview](./assets/diagrams/architecture/Arch-Mojaloop-overview-PI6.svg) +![Current Mojaloop Architecture Overview](./assets/diagrams/architecture/Arch-Mojaloop-overview-PI8.svg) _Note: Colour-grading indicates the relationship between data-store, and message-streaming / adapter-interconnects. E.g. `Central-Services` utilise `MySQL` as a Data-store, and leverage on `Kafka` for Messaging_