Merge pull request #536 from blackflux/dev

[Gally]: master <- dev
blackflux · Sep 4, 2019 · 27617ed · 27617ed
2 parents 128c751 + 2c23834
commit 27617ed
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 28 deletions.
diff --git a/package.json b/package.json
@@ -39,7 +39,7 @@
     "@babel/register": "7.5.5",
     "@blackflux/eslint-plugin-rules": "1.3.7",
     "@blackflux/robo-config-plugin": "2.6.1",
-    "aws-sdk": "2.522.0",
+    "aws-sdk": "2.523.0",
     "babel-eslint": "10.0.3",
     "chai": "4.2.0",
     "coveralls": "3.0.6",
@@ -49,7 +49,7 @@
     "eslint-plugin-json": "1.4.0",
     "eslint-plugin-markdown": "1.0.0",
     "eslint-plugin-mocha": "6.1.0",
-    "js-gardener": "2.0.81",
+    "js-gardener": "2.0.82",
     "node-tdd": "2.5.0",
     "nyc": "14.1.1",
     "semantic-release": "15.13.24"

diff --git a/src/index.js b/src/index.js
@@ -58,7 +58,7 @@ module.exports = ({ config = {}, logger = null } = {}) => {
     updateGlobalConfig: (cfg) => AWS.config.update(cfg),
     call,
     get: getService,
-    sqs: sqs(call),
+    sqs: sqs({ call, getService, logger }),
     errors
   };
 };
diff --git a/src/util/sqs.js b/src/util/sqs.js
@@ -6,7 +6,14 @@ const { SendMessageBatchError } = require('../resources/errors');
 
 const sleep = util.promisify(setTimeout);
 
-const sendBatch = async (sqsBatch, queueUrl, call, { maxRetries, backoffFunction, delaySeconds }) => {
+const sendBatch = async (sqsBatch, queueUrl, {
+  call,
+  getService,
+  maxRetries,
+  backoffFunction,
+  delaySeconds,
+  logger
+}) => {
   const pending = sqsBatch.reduce((p, msg) => {
     const id = objectHash(msg);
     return Object.assign(p, {
@@ -28,11 +35,19 @@ const sendBatch = async (sqsBatch, queueUrl, call, { maxRetries, backoffFunction
     });
     response.push(result);
     result.Successful.forEach((e) => delete pending[e.Id]);
+    if (Object.keys(pending).length !== 0 && logger !== null) {
+      logger.warn(`Failed to submit (some) message(s). Retrying: [${
+        Object
+          .values(pending)
+          .map(({ Id, MessageBody }) => `(Id=${Id}, MD5=${getService('util.crypto').md5(MessageBody, 'hex')})`)
+          .join(', ')
+      }]`);
+    }
   }
   return response;
 };
 
-module.exports = (call) => ({
+module.exports = ({ call, getService, logger }) => ({
   sendMessageBatch: async (msgs, queueUrl, {
     batchSize = 10,
     maxRetries = 10,
@@ -41,10 +56,13 @@ module.exports = (call) => ({
   } = {}) => {
     assert(batchSize <= 10, 'AWS sqs:sendMessageBatch restriction');
     const result = await Promise.all(chunk(msgs, batchSize)
-      .map((sqsBatch) => sendBatch(sqsBatch, queueUrl, call, {
+      .map((sqsBatch) => sendBatch(sqsBatch, queueUrl, {
+        call,
+        getService,
         maxRetries,
         backoffFunction,
-        delaySeconds
+        delaySeconds,
+        logger
       })));
     if (msgs.length !== result.reduce((p, c) => p + c.reduce((prev, cur) => prev + cur.Successful.length, 0), 0)) {
       throw new SendMessageBatchError(result);

diff --git a/test/util/sqs.spec.js b/test/util/sqs.spec.js
@@ -6,10 +6,10 @@ const { SendMessageBatchError } = require('../../src/resources/errors');
 describe('Testing sqs util', { useNock: true }, () => {
   let aws;
   before(() => {
-    aws = index();
+    aws = index({ logger: console });
   });
 
-  describe('Testing sendMessageBatch', () => {
+  describe('Testing sendMessageBatch', { record: console }, () => {
     it('Testing send message success', async () => {
       const result = await aws.sqs.sendMessageBatch([{
         type: 'discover',
@@ -32,7 +32,7 @@ describe('Testing sqs util', { useNock: true }, () => {
       }]]);
     });
 
-    it('Testing send message error retry', async () => {
+    it('Testing send message error retry', async ({ recorder }) => {
       const result = await aws.sqs.sendMessageBatch([{
         type: 'webhook',
         url: 'https://some-url.com/path',
@@ -63,6 +63,10 @@ describe('Testing sqs util', { useNock: true }, () => {
         }],
         Failed: []
       }]]);
+      expect(recorder.get()).to.deep.equal([
+        'Failed to submit (some) message(s). Retrying: '
+        + '[(Id=d7967cdc826c420f2482b9bac6b10b73fb156efc, MD5=90cfad0c5a2d7b4f32be02659214aaba)]'
+      ]);
     });
 
     it('Testing empty messages', async () => {

diff --git a/yarn.lock b/yarn.lock
@@ -695,10 +695,10 @@ atob@^2.1.1:
   resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
   integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
 
-aws-sdk@2.522.0:
-  version "2.522.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.522.0.tgz#f3419800fad29c03c9344d433c0782c4ad7cf0bb"
-  integrity sha512-JNUVaBqXwzDVqR/9dDw4a55aVsdDQYlf/cBM5bSj/g95wbuNWMzrY1TfAxEfSKwH0llp/1/xdXP75AKKp2UoSg==
+aws-sdk@2.523.0:
+  version "2.523.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.523.0.tgz#7467f6a41a284f61103c9d2a91eb43537ce5f166"
+  integrity sha512-x6Bam0TSG5wnn1QaBUj+Cg4Tde+xyHe35V/G58bXyhwybhrTl2ffBBFeMnO1jvmHa9aEN9p3FV2jn43e2vouZg==
   dependencies:
     buffer "4.9.1"
     events "1.1.1"
@@ -3612,10 +3612,10 @@ joi-strict@1.0.9:
   resolved "https://registry.yarnpkg.com/joi-strict/-/joi-strict-1.0.9.tgz#9b05887cbf7160e5bff6bd9a901ce0f6af42f57c"
   integrity sha512-fxky2GApiaabDQ+Itq2T5QPn7y+qpHesSHH1YcA4R81S+/zlvpweiLxbfUEP871SNS9XR+/x5XymwHSMkfbNOg==
 
-js-gardener@2.0.81:
-  version "2.0.81"
-  resolved "https://registry.yarnpkg.com/js-gardener/-/js-gardener-2.0.81.tgz#103e29e787284b3c0eeb12372dd0c1af892f9e20"
-  integrity sha512-63IrOnO52ogawNNobc+YC2Z0JF7qL/EKeeSAjFB5WkXZJ7bDTfi0YUsDYZH67alOXqJuIlKcZwrZIEq06wmHAA==
+js-gardener@2.0.82:
+  version "2.0.82"
+  resolved "https://registry.yarnpkg.com/js-gardener/-/js-gardener-2.0.82.tgz#366d7413898988cba8e4825c80f6bfe14abd6801"
+  integrity sha512-FXbo5MZ7fXuwUFMlcHNBLyl7yEf7ZbvuEAoCToPt9wNavHaQRipG0CHbceNBvA/6CFd40lLsMy+IbRnwhEBzpg==
   dependencies:
     "@hapi/joi" "15.1.1"
     chalk "2.4.2"
@@ -3627,7 +3627,7 @@ js-gardener@2.0.81:
     lodash.get "4.4.2"
     mocha "6.2.0"
     npm-check "5.9.0"
-    robo-config "3.3.2"
+    robo-config "3.3.3"
 
 js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
@@ -5077,10 +5077,10 @@ object-scan@10.0.10:
   resolved "https://registry.yarnpkg.com/object-scan/-/object-scan-10.0.10.tgz#cdce237111f29867f5ed006abf340ce4224ba4cd"
   integrity sha512-6BJQ2MhdumFdttDUAu2TN7zNPlquVDi9iix0CziLCa5QnKrlAu6Puc7Pw2/3lnEezeJ+3HZMkp9/F6K/zZjChw==
 
-object-treeify@1.1.18:
-  version "1.1.18"
-  resolved "https://registry.yarnpkg.com/object-treeify/-/object-treeify-1.1.18.tgz#568a1e917579f9158270dfefae65047c3a9b5717"
-  integrity sha512-x6vSb0QZZ8OFIjYwnL3BMvJaCK4AK/Y8oZApaF9hKBtroR6X70w4Y1ceyoP/vOHlDpqC15Hu2gq44EtDvJhkyg==
+object-treeify@1.1.19:
+  version "1.1.19"
+  resolved "https://registry.yarnpkg.com/object-treeify/-/object-treeify-1.1.19.tgz#5987fae7218c2ff0bac547ed632fa23571cc8866"
+  integrity sha512-I1RwSZyxXhgfm3+RYmBSZWozLHe1UuGn6dwW3Ppz3Iub5xzNs0TArp7UgPER22o914EIOcRfCQokRM9MLZGOOQ==
 
 object-visit@^1.0.0:
   version "1.0.1"
@@ -6245,10 +6245,10 @@ rimraf@2.6.3:
   dependencies:
     glob "^7.1.3"
 
-robo-config@3.3.2:
-  version "3.3.2"
-  resolved "https://registry.yarnpkg.com/robo-config/-/robo-config-3.3.2.tgz#a8a5493ddb1ad07207ee0c7b5210edeeedab845e"
-  integrity sha512-6+uSddS8YGbf4lUgsrWbb9St2jbLLRGxgSHElsjqmPRLiR8Bk+xWUZspKzZCrUWMk9ao+VyRKIXkNBcJZnEArA==
+robo-config@3.3.3:
+  version "3.3.3"
+  resolved "https://registry.yarnpkg.com/robo-config/-/robo-config-3.3.3.tgz#868c984423ff1b95be304ba73b0ac0a0b3362ff0"
+  integrity sha512-gkNBp/lZcC9i8KgmBf/ECG8rBt7OvSdBDmnELSb6fC1yIOHTcL/bUaN00oAUzmGxjA+Bl2Zrq2QDcGHlP96nnA==
   dependencies:
     "@hapi/joi" "15.1.1"
     app-root-path "2.2.1"
@@ -6259,7 +6259,7 @@ robo-config@3.3.2:
     lodash.get "4.4.2"
     object-deep-contain "1.0.18"
     object-scan "10.0.10"
-    object-treeify "1.1.18"
+    object-treeify "1.1.19"
     smart-fs "1.9.10"
 
 run-async@^0.1.0: