From 162ba6a5c7a6529ab071deb6b181156cbb8b672b Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Sat, 28 Sep 2024 12:06:48 +0100 Subject: [PATCH 1/4] wip: extract event data and improve logging --- README.md | 4 +- package-lock.json | 392 +++++++++++++++++++++++++++++++++++++++++++- package.json | 5 +- scripts/generate.ts | 2 +- src/index.ts | 145 ++++++++++------ 5 files changed, 491 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 2360621..8c5ebe1 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ generators: [ [ "@hookdeck/eventcatalog-generator", { - debug: true, + logLevel: "fatal" | "error" | "warn" | "info" | "debug" | "trace", connectionSourcedMatch: "regular expression string to match source names", hookdeckApiKey: "Hookdeck Project API Key. Hookdeck -> Project -> Settings -> Secrets" } @@ -55,7 +55,7 @@ npm run generate -- {flags} Supported flags are: -- `debug`: Output debug information to the console +- `log-level`: The level to log at - "fatal" | "error" | "warn" | "info" | "debug" | "trace" - `match`: Regular expression match for Source names on Connections - `dir`: Path the the Event Catalog install directory - `api-key`: Hookdeck Project API Key diff --git a/package-lock.json b/package-lock.json index 8f91f48..cbfb2da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,10 @@ "@eventcatalog/sdk": "^0.0.12", "@hookdeck/sdk": "^0.4.0", "chalk": "^4", - "minimist": "^1.2.8" + "genson-js": "^0.0.8", + "minimist": "^1.2.8", + "pino": "^9.4.0", + "pino-pretty": "^11.2.2" }, "devDependencies": { "@types/minimist": "^1.2.5", @@ -1468,6 +1471,18 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -1577,11 +1592,40 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/better-path-resolve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/better-path-resolve/-/better-path-resolve-1.0.0.tgz", @@ -1624,6 +1668,30 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/bundle-require": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.0.0.tgz", @@ -1765,6 +1833,12 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "license": "MIT" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1813,6 +1887,15 @@ "node": ">= 8" } }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -1913,6 +1996,15 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enquirer": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", @@ -2004,6 +2096,24 @@ "@types/estree": "^1.0.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -2062,6 +2172,12 @@ "node": ">=4" } }, + "node_modules/fast-copy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -2077,6 +2193,21 @@ "node": ">=8.6.0" } }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "license": "MIT" + }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -2179,6 +2310,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/genson-js": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/genson-js/-/genson-js-0.0.8.tgz", + "integrity": "sha512-4NUusDTwF+lzYh72uKV+Uvpky9iPO+YDIMpGImA5pbHfLV9HwgRCA4hYjGu78V4J4Cx2IZRTFfRERn9aUs74mw==", + "license": "Apache-2.0" + }, "node_modules/get-func-name": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", @@ -2352,6 +2489,12 @@ "node": ">= 0.4" } }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "license": "MIT" + }, "node_modules/human-id": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/human-id/-/human-id-1.0.2.tgz", @@ -2377,6 +2520,26 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2494,7 +2657,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", - "dev": true, "engines": { "node": ">=10" } @@ -2822,6 +2984,24 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", @@ -2990,6 +3170,69 @@ "node": ">=6" } }, + "node_modules/pino": { + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.4.0.tgz", + "integrity": "sha512-nbkQb5+9YPhQRz/BeQmrWpEknAaqjpAqRK8NwJpmrX/JHu7JuZC5G1CeAwJDJfGes4h+YihC6in3Q2nGb+Y09w==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.2.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^4.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.2.0.tgz", + "integrity": "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==", + "license": "MIT", + "dependencies": { + "readable-stream": "^4.0.0", + "split2": "^4.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.2.2.tgz", + "integrity": "sha512-2FnyGir8nAJAqD3srROdrF1J5BIcMT4nwj7hHSc60El6Uxlym00UbCCd8pYIterstVBFlMyF1yFV8XdGIPbj4A==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.2", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.0.0", + "pump": "^3.0.0", + "readable-stream": "^4.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", + "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==", + "license": "MIT" + }, "node_modules/pirates": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", @@ -3084,11 +3327,36 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-warning": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.0.tgz", + "integrity": "sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw==", + "license": "MIT" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -3131,6 +3399,12 @@ } ] }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, "node_modules/read-yaml-file": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/read-yaml-file/-/read-yaml-file-1.1.0.tgz", @@ -3145,6 +3419,22 @@ "node": ">=6" } }, + "node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -3157,6 +3447,15 @@ "node": ">=8.10.0" } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -3236,6 +3535,35 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -3253,6 +3581,12 @@ "node": ">=4" } }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -3341,6 +3675,15 @@ "node": ">=8" } }, + "node_modules/sonic-boom": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.1.0.tgz", + "integrity": "sha512-NGipjjRicyJJ03rPiZCJYjwlsuP2d1/5QUviozRXC7S3WdVWNK5e3Ojieb9CCyfhq2UC+3+SRd9nG3I2lPRvUw==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map": { "version": "0.8.0-beta.0", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", @@ -3425,6 +3768,15 @@ "which": "bin/which" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3442,6 +3794,15 @@ "integrity": "sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==", "dev": true }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", @@ -3550,6 +3911,18 @@ "node": ">=6" } }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sucrase": { "version": "3.35.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", @@ -3687,6 +4060,15 @@ "node": ">=0.8" } }, + "node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -4627,6 +5009,12 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", diff --git a/package.json b/package.json index 1d17949..b747c1b 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,9 @@ "@eventcatalog/sdk": "^0.0.12", "@hookdeck/sdk": "^0.4.0", "chalk": "^4", - "minimist": "^1.2.8" + "genson-js": "^0.0.8", + "minimist": "^1.2.8", + "pino": "^9.4.0", + "pino-pretty": "^11.2.2" } } diff --git a/scripts/generate.ts b/scripts/generate.ts index 01dc8ca..9ad5bc0 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -13,7 +13,7 @@ if (args.debug) { } generator(config, { - debug: args.debug, + logLevel: args['log-level'], connectionSourcedMatch: args.match, projectDir: args.dir, hookdeckApiKey: args['api-key'], diff --git a/src/index.ts b/src/index.ts index 31bc51d..f38a6ba 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,80 +2,74 @@ import utils from '@eventcatalog/sdk'; import chalk from 'chalk'; import { generateVersion } from './lib'; import { HookdeckClient } from '@hookdeck/sdk'; -import { Connection, Destination, Source } from '@hookdeck/sdk/api'; +import { Destination, Source } from '@hookdeck/sdk/api'; +import { createSchema } from 'genson-js'; +import pino from 'pino'; +import pretty from 'pino-pretty'; // The event.catalog.js values for your plugin export type EventCatalogConfig = any; // Configuration the users give your catalog export type GeneratorProps = { - debug?: boolean; + logLevel?: pino.Level; projectDir?: string; hookdeckApiKey?: string; connectionSourcedMatch?: string; }; -let _debug = false; - -const logInfo = (...args: any[]) => { - console.info(chalk.blue.apply(chalk, args)); -}; -const logSuccess = (...args: any[]) => { - console.log(chalk.green.apply(chalk, args)); -}; -const logError = (...args: any[]) => { - console.log(chalk.red.apply(chalk, args)); -}; -const logDebug = (...args: any[]) => { - if (_debug) { - console.debug.apply(console, args); - } -}; - export default async (config: EventCatalogConfig, options: GeneratorProps) => { + const stream = pretty(); + const logger = pino( + { + level: options.logLevel || 'info', + }, + stream + ); + const eventCatalogDirectory = options.projectDir || process.env.PROJECT_DIR; const hookdeckApiKey = options.hookdeckApiKey || process.env.HOOKDECK_PROJECT_API_KEY; if (!eventCatalogDirectory) { const msg = 'Please provide catalog url (env variable PROJECT_DIR)'; - logError(msg); + logger.error(msg); throw new Error(msg); } if (!hookdeckApiKey) { const msg = 'Please provide Hookdeck Project API Key (env variable HOOKDECK_PROJECT_API_KEY)'; - logError(msg); + logger.error(msg); throw new Error(msg); } - _debug = options.debug || false; - const hookdeckClient = new HookdeckClient({ token: hookdeckApiKey }); const connectionsResponse = await hookdeckClient.connection.list(); if (connectionsResponse.models !== undefined && connectionsResponse.models.length === 0) { - logInfo('No connections found'); + logger.info('No connections found'); return; } let connections = connectionsResponse.models!; const connectionSourceMatch = options.connectionSourcedMatch ? new RegExp(options.connectionSourcedMatch) : undefined; if (connectionSourceMatch) { - logInfo(`Applying Connection Source Match: "${connectionSourceMatch}"`); - logDebug(connectionSourceMatch); + logger.info(`Applying Connection Source Match: "${connectionSourceMatch}"`); + logger.debug(connectionSourceMatch); connections = connections.filter((c) => { if (connectionSourceMatch && connectionSourceMatch.test(c.source.name) === false) { - logDebug(`Connection "${c.source.name}" does not match "${connectionSourceMatch}"`); + logger.debug(`Connection "${c.source.name}" does not match "${connectionSourceMatch}"`); return false; } return true; }); } - logInfo(`Found ${connections.length} connections`); - logDebug( - `Generating Event Catalog for ${connections.length} Connections with Sources: \n${connections.map((c) => `- ${c.source.name}\n`)}` + logger.info(`Found ${connections.length} connections`); + logger.debug( + `Generating Event Catalog for ${connections.length} Connections with Sources: \n${connections + .map((c) => `- ${c.source.name}`) + .join('\n')}` ); const sources: { [key: string]: Source } = {}; @@ -87,7 +81,7 @@ export default async (config: EventCatalogConfig, options: GeneratorProps) => { } // if (options.debug) { - // logDebug('Exiting early due to debug flag'); + // logger.debug('Exiting early due to debug flag'); // return; // } @@ -107,47 +101,96 @@ export default async (config: EventCatalogConfig, options: GeneratorProps) => { markdown: source.description || '', }); } else { - logDebug(`Service for Source ${source.name} already exists`); + logger.debug(`Service for Source ${source.name} already exists`); } const requests = await hookdeckClient.request.list({ sourceId: source.id }); if (requests.models) { - logDebug(`Found ${requests.models.length} Requests for Source ${source.id}`); + logger.debug(`Found ${requests.models.length} Requests for Source ${source.id}`); for (let i = 0; i < requests.models.length; ++i) { const request = requests.models[i]; - // TODO: extract schema from request - // https://www.npmjs.com/package/genson-js - // TODO: determine a way to generate an ID for the event - - const eventId = `${source.id}:${i}`; + const fullRequest = await hookdeckClient.request.retrieve(request.id); + let eventType = `${source.id}:${i}`; + let schema = undefined; + + if (fullRequest.data) { + // Create schema + logger.trace(`Request ID: ${request.id}`, JSON.stringify(fullRequest.data)); + try { + schema = createSchema(fullRequest.data.body); + logger.trace(`Schema for Request ID: ${request.id}`, JSON.stringify(schema)); + } catch (e) { + logger.error(`Error generating schema for Request ID: ${request.id}`, e); + } + + // Try to determine an event type + if (fullRequest.data.body && typeof fullRequest.data.body === 'object') { + if ('type' in fullRequest.data.body) { + eventType = fullRequest.data.body.type as string; + } else if ('eventType' in fullRequest.data.body) { + eventType = fullRequest.data.body.eventType as string; + } else { + logger.warn(`Could not determine event type. No 'type' or 'eventType' field found in Request ID: ${request.id}`); + } + } + } else { + logger.error(`fullRequest.data is undefined for request ID: ${request.id}`); + } const eventVersion = generateVersion(request.createdAt); - const existingEvent = await getEvent(eventId, eventVersion); + // TODO: apply areSchemasEqual logic from genson-js + const existingEvent = await getEvent(eventType, eventVersion); if (!existingEvent) { + // EventCatalog does not support "." in event IDs + const eventId = eventType.replace('.', ':'); await writeEvent({ id: eventId, - markdown: `Example: - ${JSON.stringify(request.data, null, 2)}`, - name: eventId, + markdown: ` +### Schema + +\`\`\`json +${JSON.stringify(schema, null, 2)} +\`\`\` + +### Example + +#### Body + +\`\`\`json +${JSON.stringify(fullRequest.data?.body, null, 2)} +\`\`\` + +#### Headers + +\`\`\`json +${JSON.stringify(fullRequest.data?.headers, null, 2)} +\`\`\` +`, + name: eventType, version: eventVersion, }); - await addEventToService(source.id, 'receives', { + await addEventToService(source.id, 'sends', { id: eventId, version: eventVersion, }); - logDebug(`Written event for Request: ${JSON.stringify(request)}`); + logger.debug(`Written event for Request: ${JSON.stringify(request)}`); } else { - logDebug(`Event ${eventId} already exists`); + logger.debug(`Event ${eventType} already exists`); } } } } - logSuccess(`Created Services for ${Object.keys(sources).length} Sources`); + logger.info(chalk.green(`Created Services for ${Object.keys(sources).length} Sources`)); + + if (options.logLevel === 'trace') { + logger.debug('Exiting early due to trace flag'); + return; + } // Create a Service for each Destination for (const destination of Object.values(destinations)) { @@ -162,12 +205,12 @@ export default async (config: EventCatalogConfig, options: GeneratorProps) => { markdown: destination.description || '', }); } else { - logDebug(`Service or Destination ${destination.name} already exists`); + logger.debug(`Service or Destination ${destination.name} already exists`); } const events = await hookdeckClient.event.list({ destinationId: destination.id }); if (events.models) { - logDebug(`Found ${events.models.length} Events for Destination ${destination.id}`); + logger.debug(`Found ${events.models.length} Events for Destination ${destination.id}`); for (let i = 0; i < events.models.length; ++i) { const event = events.models[i]; @@ -193,13 +236,13 @@ export default async (config: EventCatalogConfig, options: GeneratorProps) => { version: eventVersion, }); - logDebug(`Written event for Event: ${JSON.stringify(event)}`); + logger.debug(`Written event for Event: ${JSON.stringify(event)}`); } else { - logDebug(`Event ${eventId} already exists`); + logger.debug(`Event ${eventId} already exists`); } } } } - logSuccess(`Created Services for ${Object.keys(destinations).length} Destinations`); + logger.info(chalk.green(`Created Services for ${Object.keys(destinations).length} Destinations`)); }; From 5d023ab35d017f3697cefb71cd3560e369b10a41 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 30 Sep 2024 15:25:51 +0100 Subject: [PATCH 2/4] chore: set event ID separator --- src/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index f38a6ba..ad29c19 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,8 @@ export type GeneratorProps = { connectionSourcedMatch?: string; }; +const EVENT_ID_SEPARATOR = '-'; + export default async (config: EventCatalogConfig, options: GeneratorProps) => { const stream = pretty(); const logger = pino( @@ -144,7 +146,7 @@ export default async (config: EventCatalogConfig, options: GeneratorProps) => { const existingEvent = await getEvent(eventType, eventVersion); if (!existingEvent) { // EventCatalog does not support "." in event IDs - const eventId = eventType.replace('.', ':'); + const eventId = eventType.replace('.', EVENT_ID_SEPARATOR); await writeEvent({ id: eventId, markdown: ` @@ -218,7 +220,7 @@ ${JSON.stringify(fullRequest.data?.headers, null, 2)} // TODO: extract schema from event // TODO: determine a way to generate an ID for the event - const eventId = `${event.id}:${i}`; + const eventId = `${event.id}${EVENT_ID_SEPARATOR}${i}`; const eventVersion = generateVersion(event.createdAt); const existingEvent = await getEvent(eventId, eventVersion); From 767482832e9d96da781a1af21950286f307bb693 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 30 Sep 2024 15:26:11 +0100 Subject: [PATCH 3/4] fix: EventCatalog requires versions to be in SemVer format --- src/lib.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib.ts b/src/lib.ts index 10c8611..e275077 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -2,8 +2,14 @@ function pad(val: any) { return (val + '').padStart(2, '0'); } +const VERSION_SEPARATOR = ''; + export function generateVersion(date?: Date) { const generateTime = date ?? new Date(); - const version = `${generateTime.getFullYear()}-${pad(generateTime.getMonth() + 1)}${pad(generateTime.getDate())}-${pad(generateTime.getHours())}${pad(generateTime.getMinutes())}${pad(generateTime.getSeconds())}`; + + const dateVersionMinor = `${generateTime.getFullYear()}${VERSION_SEPARATOR}${pad(generateTime.getMonth() + 1)}${pad(generateTime.getDate())}`; + const dateVersionPatch = `${pad(generateTime.getHours())}${pad(generateTime.getMinutes())}${pad(generateTime.getSeconds())}`; + const version = `0.${dateVersionMinor}.${dateVersionPatch}`; + return version; } From d113a0d32f66281b841beca4c977d907cf0d62e3 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 30 Sep 2024 23:45:24 +0100 Subject: [PATCH 4/4] feat: full support for services, destinations, sources, and request + event schemas --- README.md | 7 + scripts/generate.ts | 2 + src/index.ts | 528 +++++++++++++++++++++++++++++--------------- src/lib.ts | 4 + 4 files changed, 366 insertions(+), 175 deletions(-) diff --git a/README.md b/README.md index 8c5ebe1..e595d4d 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,13 @@ Supported flags are: - `match`: Regular expression match for Source names on Connections - `dir`: Path the the Event Catalog install directory - `api-key`: Hookdeck Project API Key +- `max-events`: The maximum number of Requests/Events to process per Source/Destination + +Example: + +```sh +npm run generate -- --log-level debug --match "stripe-production" --domain Payments +``` The `generate` script will also use the following environment variables: diff --git a/scripts/generate.ts b/scripts/generate.ts index 9ad5bc0..0871cee 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -17,4 +17,6 @@ generator(config, { connectionSourcedMatch: args.match, projectDir: args.dir, hookdeckApiKey: args['api-key'], + domain: args.domain, + processMaxEvents: args['max-events'], }); diff --git a/src/index.ts b/src/index.ts index ad29c19..b981f78 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,9 @@ import utils from '@eventcatalog/sdk'; import chalk from 'chalk'; -import { generateVersion } from './lib'; +import { generateVersion, sleep } from './lib'; import { HookdeckClient } from '@hookdeck/sdk'; import { Destination, Source } from '@hookdeck/sdk/api'; -import { createSchema } from 'genson-js'; +import { createSchema, Schema } from 'genson-js'; import pino from 'pino'; import pretty from 'pino-pretty'; @@ -16,140 +16,219 @@ export type GeneratorProps = { projectDir?: string; hookdeckApiKey?: string; connectionSourcedMatch?: string; + domain?: string; + processMaxEvents?: number; }; const EVENT_ID_SEPARATOR = '-'; +const SLEEP_TIME = 200; + +class Generator { + config: EventCatalogConfig; + options: GeneratorProps; + eventCatalogDirectory: string; + hookdeckApiKey: string; + generationRunDate: Date; + processMaxEvents: number; + logger: any; + hookdeckClient: HookdeckClient; + + constructor(config: EventCatalogConfig, options: GeneratorProps) { + this.config = config; + this.options = options; + + const eventCatalogDirectory = options.projectDir || process.env.PROJECT_DIR; + const hookdeckApiKey = options.hookdeckApiKey || process.env.HOOKDECK_PROJECT_API_KEY; + + const stream = pretty(); + this.logger = pino( + { + level: options.logLevel || 'info', + }, + stream + ); + + if (!eventCatalogDirectory) { + const msg = 'Please provide catalog url (env variable PROJECT_DIR)'; + this.logger.error(msg); + throw new Error(msg); + } -export default async (config: EventCatalogConfig, options: GeneratorProps) => { - const stream = pretty(); - const logger = pino( - { - level: options.logLevel || 'info', - }, - stream - ); - - const eventCatalogDirectory = options.projectDir || process.env.PROJECT_DIR; - const hookdeckApiKey = options.hookdeckApiKey || process.env.HOOKDECK_PROJECT_API_KEY; - - if (!eventCatalogDirectory) { - const msg = 'Please provide catalog url (env variable PROJECT_DIR)'; - logger.error(msg); - throw new Error(msg); - } + if (!hookdeckApiKey) { + const msg = 'Please provide Hookdeck Project API Key (env variable HOOKDECK_PROJECT_API_KEY)'; + this.logger.error(msg); + throw new Error(msg); + } - if (!hookdeckApiKey) { - const msg = 'Please provide Hookdeck Project API Key (env variable HOOKDECK_PROJECT_API_KEY)'; - logger.error(msg); - throw new Error(msg); - } + this.eventCatalogDirectory = eventCatalogDirectory; + this.hookdeckApiKey = hookdeckApiKey; - const hookdeckClient = new HookdeckClient({ token: hookdeckApiKey }); + this.hookdeckClient = new HookdeckClient({ token: this.hookdeckApiKey }); - const connectionsResponse = await hookdeckClient.connection.list(); - if (connectionsResponse.models !== undefined && connectionsResponse.models.length === 0) { - logger.info('No connections found'); - return; + // For the moment, versions are created for each generation run + // In the future: + // 1. a version could be inferred from the schema though this has been tested + // and proved to be unreliable with property values being null or a string signalling a new schema. + // 2. a version could be passed in as a parameter to the generator + this.generationRunDate = new Date(); + this.processMaxEvents = options.processMaxEvents || 200; } - let connections = connectionsResponse.models!; - const connectionSourceMatch = options.connectionSourcedMatch ? new RegExp(options.connectionSourcedMatch) : undefined; - if (connectionSourceMatch) { - logger.info(`Applying Connection Source Match: "${connectionSourceMatch}"`); - logger.debug(connectionSourceMatch); + async generate() { + const { config, options } = this; - connections = connections.filter((c) => { - if (connectionSourceMatch && connectionSourceMatch.test(c.source.name) === false) { - logger.debug(`Connection "${c.source.name}" does not match "${connectionSourceMatch}"`); - return false; - } - return true; - }); - } + // EventCatalog SDK (https://www.eventcatalog.dev/docs/sdk) + const { writeService, getService, writeEvent, getEvent, addEventToService, addServiceToDomain, writeDomain } = utils( + this.eventCatalogDirectory + ); - logger.info(`Found ${connections.length} connections`); - logger.debug( - `Generating Event Catalog for ${connections.length} Connections with Sources: \n${connections - .map((c) => `- ${c.source.name}`) - .join('\n')}` - ); + if (options.domain) { + writeDomain({ + id: options.domain, + name: options.domain, + version: generateVersion(this.generationRunDate), + markdown: '', + }); + } - const sources: { [key: string]: Source } = {}; - const destinations: { [key: string]: Destination } = {}; + const connectionsResponse = await this.hookdeckClient.connection.list(); + if (connectionsResponse.models !== undefined && connectionsResponse.models.length === 0) { + this.logger.info('No connections found'); + return; + } - for (const connection of connections) { - sources[connection.source.id] = connection.source; - destinations[connection.destination.id] = connection.destination; - } + let connections = connectionsResponse.models!; + const connectionSourceMatch = options.connectionSourcedMatch ? new RegExp(options.connectionSourcedMatch) : undefined; + if (connectionSourceMatch) { + this.logger.info(`Applying Connection Source Match: "${connectionSourceMatch}"`); + this.logger.debug(connectionSourceMatch); - // if (options.debug) { - // logger.debug('Exiting early due to debug flag'); - // return; - // } - - // EventCatalog SDK (https://www.eventcatalog.dev/docs/sdk) - const { writeService, getService, writeEvent, getEvent, addEventToService } = utils(eventCatalogDirectory); - - // Create a Service for each Source - for (const source of Object.values(sources)) { - const serviceVersion = generateVersion(source.updatedAt); - const existingSourceService = await getService(source.id, serviceVersion); - - if (!existingSourceService) { - await writeService({ - id: source.id, - name: source.name, - version: serviceVersion, - markdown: source.description || '', + connections = connections.filter((c) => { + if (connectionSourceMatch && connectionSourceMatch.test(c.source.name) === false) { + this.logger.debug(`Connection "${c.source.name}" does not match "${connectionSourceMatch}"`); + return false; + } + return true; }); - } else { - logger.debug(`Service for Source ${source.name} already exists`); } - const requests = await hookdeckClient.request.list({ sourceId: source.id }); - if (requests.models) { - logger.debug(`Found ${requests.models.length} Requests for Source ${source.id}`); - - for (let i = 0; i < requests.models.length; ++i) { - const request = requests.models[i]; - - const fullRequest = await hookdeckClient.request.retrieve(request.id); - let eventType = `${source.id}:${i}`; - let schema = undefined; - - if (fullRequest.data) { - // Create schema - logger.trace(`Request ID: ${request.id}`, JSON.stringify(fullRequest.data)); - try { - schema = createSchema(fullRequest.data.body); - logger.trace(`Schema for Request ID: ${request.id}`, JSON.stringify(schema)); - } catch (e) { - logger.error(`Error generating schema for Request ID: ${request.id}`, e); - } + this.logger.info(`Found ${connections.length} connections`); + this.logger.debug( + `Generating Event Catalog for ${connections.length} Connections with Sources: \n${connections + .map((c) => `- ${c.source.name}`) + .join('\n')}` + ); + + const sources: { [key: string]: Source } = {}; + const destinations: { [key: string]: Destination } = {}; + + for (const connection of connections) { + sources[connection.source.id] = connection.source; + destinations[connection.destination.id] = connection.destination; + } + + // if (options.debug) { + // this.logger.debug('Exiting early due to debug flag'); + // return; + // } + + await this.processSources(sources); + + await this.processDestinations(destinations); + } + + private async processDestinations(destinations: { [key: string]: Destination }) { + const { writeService, getService, writeEvent, getEvent, addEventToService, addServiceToDomain, writeDomain } = utils( + this.eventCatalogDirectory + ); + for (const destination of Object.values(destinations)) { + // const destinationVersion = generateVersion(destination.updatedAt); + const destinationVersion = generateVersion(this.generationRunDate); + const existingSourceService = await getService(destination.id, destinationVersion); + + if (!existingSourceService) { + // Create a Service for each Source + await writeService({ + id: destination.id, + name: destination.name, + version: destinationVersion, + markdown: destination.description || '', + }); + + if (this.options.domain) { + await addServiceToDomain(this.options.domain, { + id: destination.id, + version: destinationVersion, + }); + } + } else { + this.logger.debug(`Service or Destination ${destination.name} already exists`); + } - // Try to determine an event type - if (fullRequest.data.body && typeof fullRequest.data.body === 'object') { - if ('type' in fullRequest.data.body) { - eventType = fullRequest.data.body.type as string; - } else if ('eventType' in fullRequest.data.body) { - eventType = fullRequest.data.body.eventType as string; + let nextEvent = undefined; + let eventIteration = 1; + const processedEvents = new Map(); + do { + const events = await this.hookdeckClient.event.list({ destinationId: destination.id, next: nextEvent }); + if (events.models) { + this.logger.debug(`Found ${events.models.length} Events for Destination ${destination.id}`); + + for (let i = 0; i < events.models.length; ++i) { + const event = events.models[i]; + + if (processedEvents.has(event.id)) { + throw new Error(`Event ID ${event.id} has already been processed`); + } + processedEvents.set(event.id, true); + + // Try to avoid rate limiting + await sleep(SLEEP_TIME); + + const fullEvent = await this.hookdeckClient.event.retrieve(event.id); + let eventType = `${destination.id}:${i}`; + let schema = undefined; + + if (fullEvent.data) { + this.logger.debug( + `Processing Event ID "${event.id}". ${i + 1} of ${events.models.length}. Iteration: ${eventIteration}. Processed ${processedEvents.size} of a maximum ${this.processMaxEvents} events.` + ); + + // Create schema + this.logger.trace(`Event ID: ${fullEvent.id} %s`, JSON.stringify(fullEvent.data)); + try { + schema = createSchema(fullEvent.data.body); + this.logger.trace(`Schema for Event ID: ${fullEvent.id} %s`, JSON.stringify(schema)); + } catch (e) { + this.logger.error(`Error generating schema for Event ID: ${fullEvent.id} %s`, e); + } + + // Try to determine an event type + if (fullEvent.data.body && typeof fullEvent.data.body === 'object') { + if ('type' in fullEvent.data.body) { + eventType = fullEvent.data.body.type as string; + } else if ('eventType' in fullEvent.data.body) { + eventType = fullEvent.data.body.eventType as string; + } else { + this.logger.warn( + `Could not determine event type. No 'type' or 'eventType' field found in Event ID: ${event.id}` + ); + } + } } else { - logger.warn(`Could not determine event type. No 'type' or 'eventType' field found in Request ID: ${request.id}`); + this.logger.error(`fullEvent.data is undefined for Event ID: ${event.id}`); } - } - } else { - logger.error(`fullRequest.data is undefined for request ID: ${request.id}`); - } - const eventVersion = generateVersion(request.createdAt); - // TODO: apply areSchemasEqual logic from genson-js - const existingEvent = await getEvent(eventType, eventVersion); - if (!existingEvent) { - // EventCatalog does not support "." in event IDs - const eventId = eventType.replace('.', EVENT_ID_SEPARATOR); - await writeEvent({ - id: eventId, - markdown: ` + // const eventVersion = generateVersion(fullEvent.createdAt); + const eventVersion = generateVersion(this.generationRunDate); + const existingEvent = await getEvent(eventType, eventVersion); + + // EventCatalog does not support "." in event IDs + const eventId = eventType.replace('.', EVENT_ID_SEPARATOR); + + if (!existingEvent) { + await writeEvent({ + id: eventId, + markdown: ` ### Schema \`\`\`json @@ -161,90 +240,189 @@ ${JSON.stringify(schema, null, 2)} #### Body \`\`\`json -${JSON.stringify(fullRequest.data?.body, null, 2)} +${JSON.stringify(fullEvent.data?.body, null, 2)} \`\`\` #### Headers \`\`\`json -${JSON.stringify(fullRequest.data?.headers, null, 2)} +${JSON.stringify(fullEvent.data?.headers, null, 2)} \`\`\` + +### Meta + +Event ID: ${fullEvent.id} `, - name: eventType, - version: eventVersion, - }); + name: eventType, + version: eventVersion, + }); - await addEventToService(source.id, 'sends', { - id: eventId, - version: eventVersion, - }); + this.logger.debug(`Written event for Event: ${JSON.stringify(fullEvent)}`); + } else { + this.logger.debug(`Event ${eventType} already exists`); + } + + await addEventToService(destination.id, 'receives', { + id: eventId, + version: eventVersion, + }); - logger.debug(`Written event for Request: ${JSON.stringify(request)}`); - } else { - logger.debug(`Event ${eventType} already exists`); + this.logger.trace(`Registered eventType ${eventType} for service ${destination.name} for Event ID: ${event.id}`); + } } - } + + nextEvent = events.pagination?.next; + ++eventIteration; + } while (processedEvents.size < this.processMaxEvents && nextEvent !== undefined); } + + this.logger.info(chalk.green(`Created Services for ${Object.keys(destinations).length} Destinations`)); } - logger.info(chalk.green(`Created Services for ${Object.keys(sources).length} Sources`)); + private async processSources(sources: { [key: string]: Source }) { + const { writeService, getService, writeEvent, getEvent, addEventToService, addServiceToDomain } = utils( + this.eventCatalogDirectory + ); + + for (const source of Object.values(sources)) { + // const serviceVersion = generateVersion(source.updatedAt); + const serviceVersion = generateVersion(this.generationRunDate); + const existingSourceService = await getService(source.id, serviceVersion); + + if (!existingSourceService) { + // Create a Service for each Destination + await writeService({ + id: source.id, + name: source.name, + version: serviceVersion, + markdown: source.description || '', + }); + + if (this.options.domain) { + await addServiceToDomain(this.options.domain, { + id: source.id, + version: serviceVersion, + }); + } + } else { + this.logger.debug(`Service for Source ${source.name} already exists`); + } - if (options.logLevel === 'trace') { - logger.debug('Exiting early due to trace flag'); - return; - } + let nextRequest = undefined; + let requestIteration = 1; + const processedRequests = new Map(); + do { + const requests = await this.hookdeckClient.request.list({ sourceId: source.id, next: nextRequest }); + if (requests.models) { + this.logger.debug(`Found ${requests.models.length} Requests for Source ${source.id}`); - // Create a Service for each Destination - for (const destination of Object.values(destinations)) { - const destinationVersion = generateVersion(destination.updatedAt); - const existingSourceService = await getService(destination.id, destinationVersion); - - if (!existingSourceService) { - await writeService({ - id: destination.id, - name: destination.name, - version: generateVersion(destination.updatedAt), - markdown: destination.description || '', - }); - } else { - logger.debug(`Service or Destination ${destination.name} already exists`); - } + for (let i = 0; i < requests.models.length; ++i) { + const request = requests.models[i]; - const events = await hookdeckClient.event.list({ destinationId: destination.id }); - if (events.models) { - logger.debug(`Found ${events.models.length} Events for Destination ${destination.id}`); + if (processedRequests.has(request.id)) { + throw new Error(`Request ID ${request.id} has already been processed`); + } + processedRequests.set(request.id, true); + + // Try to avoid rate limiting + await sleep(SLEEP_TIME); + + const fullRequest = await this.hookdeckClient.request.retrieve(request.id, { maxRetries: 1 }); + let eventType = `${source.id}:${i}`; + let schema: Schema | undefined = undefined; + + if (fullRequest.data) { + // Create schema + this.logger.debug( + `Processing Request ID "${request.id}". ${i + 1} of ${requests.models.length}. Iteration: ${requestIteration}. Processed ${processedRequests.size} of a maximum ${this.processMaxEvents} requests.` + ); + this.logger.trace(`Request ID: ${request.id} %s`, JSON.stringify(fullRequest.data)); + try { + schema = createSchema(fullRequest.data.body); + this.logger.trace(`Schema for Request ID: ${request.id} %s`, JSON.stringify(schema)); + } catch (e) { + this.logger.error(`Error generating schema for Request ID: ${request.id} %s`, e); + } + + // Try to determine an event type + if (fullRequest.data.body && typeof fullRequest.data.body === 'object') { + if ('type' in fullRequest.data.body) { + eventType = fullRequest.data.body.type as string; + } else if ('eventType' in fullRequest.data.body) { + eventType = fullRequest.data.body.eventType as string; + } else { + this.logger.warn( + `Could not determine event type. No 'type' or 'eventType' field found in Request ID: ${request.id}` + ); + } + } + } else { + this.logger.error(`fullRequest.data is undefined for request ID: ${request.id}`); + throw new Error(`fullRequest.data is undefined for request ID: ${request.id}`); + } - for (let i = 0; i < events.models.length; ++i) { - const event = events.models[i]; + // const eventVersion = generateVersion(request.createdAt); + const eventVersion = generateVersion(this.generationRunDate); + const existingEvent = await getEvent(eventType, eventVersion); - // TODO: extract schema from event - // TODO: determine a way to generate an ID for the event + // EventCatalog does not support "." in event IDs + const eventId = eventType.replace('.', EVENT_ID_SEPARATOR); - const eventId = `${event.id}${EVENT_ID_SEPARATOR}${i}`; + if (!existingEvent) { + await writeEvent({ + id: eventId, + markdown: ` +### Schema - const eventVersion = generateVersion(event.createdAt); - const existingEvent = await getEvent(eventId, eventVersion); - if (!existingEvent) { - await writeEvent({ - id: eventId, - markdown: `Example: - ${JSON.stringify(event.data, null, 2)}`, - name: eventId, - version: eventVersion, - }); +\`\`\`json +${JSON.stringify(schema, null, 2)} +\`\`\` - await addEventToService(destination.id, 'receives', { - id: eventId, - version: eventVersion, - }); +### Example - logger.debug(`Written event for Event: ${JSON.stringify(event)}`); - } else { - logger.debug(`Event ${eventId} already exists`); +#### Body + +\`\`\`json +${JSON.stringify(fullRequest.data?.body, null, 2)} +\`\`\` + +#### Headers + +\`\`\`json +${JSON.stringify(fullRequest.data?.headers, null, 2)} +\`\`\` + +### Meta + +Request ID: ${fullRequest.id} +`, + name: eventType, + version: eventVersion, + }); + + this.logger.debug(`Written event for Request: ${JSON.stringify(request)}`); + + await addEventToService(source.id, 'sends', { + id: eventId, + version: eventVersion, + }); + + this.logger.trace(`Registered eventType ${eventType} for service ${source.name} for Request ID: ${request.id}`); + } else { + this.logger.debug(`Event ${eventType} already exists`); + } + } } - } + nextRequest = requests.pagination?.next; + ++requestIteration; + } while (processedRequests.size < this.processMaxEvents && nextRequest !== undefined); } + + this.logger.info(chalk.green(`Created Services for ${Object.keys(sources).length} Sources`)); } +} - logger.info(chalk.green(`Created Services for ${Object.keys(destinations).length} Destinations`)); +export default async (config: EventCatalogConfig, options: GeneratorProps) => { + const generator = new Generator(config, options); + await generator.generate(); }; diff --git a/src/lib.ts b/src/lib.ts index e275077..1c4f5ad 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -13,3 +13,7 @@ export function generateVersion(date?: Date) { return version; } + +export function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +}