diff --git a/README.md b/README.md index e580f84..314483a 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ A 4th parameter is a configuration object: * `delay: number` adjust the delay in millisecond between two scan * `dyCapacityUnitLimit: number` defined a dynamodb capacity unit limit to stop scan if it's reached * `progressCallbackInterval: number` interval in millisecond for calling progress callback +* `primaryKeyName: string` the name of your primary key (default: 'uuid') +* `primaryKeyType: string` the dynamoDB type of your primary key (default: 'S') then you can call the `run` method to start the scan stream to kinesis. it returns a promise where your callback will call at the end of a full scan. diff --git a/package-lock.json b/package-lock.json index 44318ed..2604f90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,24 @@ { - "name": "dynamodb-to-kinesis", - "version": "0.1.0", + "name": "dyki", + "version": "1.0.0", "lockfileVersion": 1, + "requires": true, "dependencies": { "aws-sdk": { - "version": "2.84.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.84.0.tgz", - "integrity": "sha1-yHuwW8Q76mgcEVUD9zp+eEyb/rY=" + "version": "2.697.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.697.0.tgz", + "integrity": "sha512-aNrwiPRHQyzjJxpfgLwVOevuGTOMkU5uiP4VDIngfc/k4s2kQgLSyhLSKmNTjbubHCHfs1sQQkP3RXK2Oi8Rbw==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } }, "base-64": { "version": "0.1.0", @@ -14,19 +26,19 @@ "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" }, "base64-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", - "integrity": "sha512-dwVUVIXsBZXwTuwnXI9RK8sBmgq09NDHzyR9SAph9eqk76gKK2JSQmZARC2zRC81JC2QTtxD0ARU5qTS25gIGw==" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "buffer": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=" - }, - "crypto-browserify": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-1.0.9.tgz", - "integrity": "sha1-zFRJaF37hesRyYKKzHy4erW7/MA=" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } }, "events": { "version": "1.1.1", @@ -34,9 +46,9 @@ "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "isarray": { "version": "1.0.0", @@ -48,11 +60,6 @@ "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, - "lodash": { - "version": "4.17.4", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", - "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" - }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -71,12 +78,16 @@ "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=" + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } }, "uuid": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz", - "integrity": "sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE=" + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, "when": { "version": "3.7.8", @@ -84,14 +95,18 @@ "integrity": "sha1-xxMLan6gRpPoQs3J56Hyqjmjn4I=" }, "xml2js": { - "version": "0.4.17", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", - "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=" + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } }, "xmlbuilder": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", - "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=" + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" } } } diff --git a/package.json b/package.json index 00677e9..07db154 100644 --- a/package.json +++ b/package.json @@ -1,38 +1,42 @@ { - "name": "dyki", - "version": "1.0.0", - "description": "Scan DynamoDB items to dispatch into Kinesis Stream", - "keywords": [ - "aws", - "serverless", - "amazon web service", - "dynamodb", - "kinesis", - "stream", - "elastic" - ], - "repository": { - "type": "git", - "url": "https://github.com/pierozi/dyki" - }, - "bugs": "https://github.com/pierozi/dyki/issues", - "maintainers": [{ - "name": "Pierre Tomasina", - "web": "http://continuous.lu" - }], - "contributors": [{ - "name": "Pierre Tomasina", - "web": "http://continuous.lu" - }], - "main": "src/main.js", - "scripts": { - "test": "node tests/scratch.js" - }, - "author": "Pierre Tomasina (continuous.lu)", - "license": "MIT", - "dependencies": { - "aws-sdk": "^2.84.0", - "base-64": "^0.1.0", - "when": "^3.7.8" - } + "name": "dyki", + "version": "1.0.0", + "description": "Scan DynamoDB items to dispatch into Kinesis Stream", + "keywords": [ + "aws", + "serverless", + "amazon web service", + "dynamodb", + "kinesis", + "stream", + "elastic" + ], + "repository": { + "type": "git", + "url": "https://github.com/pierozi/dyki" + }, + "bugs": "https://github.com/pierozi/dyki/issues", + "maintainers": [ + { + "name": "Pierre Tomasina", + "web": "http://continuous.lu" + } + ], + "contributors": [ + { + "name": "Pierre Tomasina", + "web": "http://continuous.lu" + } + ], + "main": "src/main.js", + "scripts": { + "test": "node tests/scratch.js" + }, + "author": "Pierre Tomasina (continuous.lu)", + "license": "MIT", + "dependencies": { + "aws-sdk": "^2.697.0", + "base-64": "^0.1.0", + "when": "^3.7.8" + } } diff --git a/src/main.js b/src/main.js index 05e4f1c..9b7c086 100644 --- a/src/main.js +++ b/src/main.js @@ -9,6 +9,8 @@ DyKi.Config = { dyCapacityUnitLimit: 5, delay: 1000, progressCallbackInterval: 1000, + primaryKeyName: 'uuid', + primaryKeyType: 'S', }; DyKi.Error = class extends Error {}; @@ -68,7 +70,7 @@ DyKi.Client = class { } clearInterval(progressInterval); - + if (this.EOF) { return when(this.info()); } @@ -135,7 +137,7 @@ DyKi.Client = class { const params = { Data: JSON.stringify(record), - PartitionKey: item.uuid['S'], + PartitionKey: item[this.config.primaryKeyName][this.config.primaryKeyType], StreamName: this.streamName, };