feat: 🎸 add noerror option in URLConnect

Inist-CNRS · Apr 8, 2021 · a70909b · a70909b
1 parent d2e1ef1
commit a70909b
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 26 deletions.
diff --git a/docs/plugin-basics.md b/docs/plugin-basics.md
@@ -18,8 +18,8 @@ npm install @ezs/basics
 -   [OBJFlatten](#objflatten)
 -   [TXTZip](#txtzip)
 -   [OBJCount](#objcount)
--   [URLConnect](#urlconnect)
 -   [URLString](#urlstring)
+-   [URLConnect](#urlconnect)
 -   [URLFetch](#urlfetch)
 -   [URLStream](#urlstream)
 -   [BUFObject](#bufobject)
@@ -83,6 +83,12 @@ Take `Object` and count how many objects are received and sent the total
 
 Returns **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** 
 
+### URLString
+
+Take `Object` representing a URL and stringify it
+
+Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
+
 ### URLConnect
 
 Take `Object` and send it to an URL
@@ -92,16 +98,11 @@ the output will be the content of URL
 
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Pasre as JSON the content of URL (optional, default `false`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+-   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
-### URLString
-
-Take `Object` representing a URL and stringify it
-
-Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
-
 ### URLFetch
 
 Take `Object` and create a new field with the content of URL.
@@ -112,7 +113,7 @@ Or if no target will be specified, the output will be the content of URL
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth
 -   `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of URL (optional, default `false`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
@@ -126,7 +127,7 @@ Take `Object` as parameters of URL, throw each chunk from the result
 
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch (by default input string is taken)
 -   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose the path to split JSON result (optional, default `*`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 

diff --git a/packages/basics/README.md b/packages/basics/README.md
@@ -18,8 +18,8 @@ npm install @ezs/basics
 -   [OBJFlatten](#objflatten)
 -   [TXTZip](#txtzip)
 -   [OBJCount](#objcount)
--   [URLConnect](#urlconnect)
 -   [URLString](#urlstring)
+-   [URLConnect](#urlconnect)
 -   [URLFetch](#urlfetch)
 -   [URLStream](#urlstream)
 -   [BUFObject](#bufobject)
@@ -83,6 +83,12 @@ Take `Object` and count how many objects are received and sent the total
 
 Returns **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** 
 
+### URLString
+
+Take `Object` representing a URL and stringify it
+
+Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
+
 ### URLConnect
 
 Take `Object` and send it to an URL
@@ -92,16 +98,11 @@ the output will be the content of URL
 
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Pasre as JSON the content of URL (optional, default `false`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+-   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
-### URLString
-
-Take `Object` representing a URL and stringify it
-
-Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
-
 ### URLFetch
 
 Take `Object` and create a new field with the content of URL.
@@ -112,7 +113,7 @@ Or if no target will be specified, the output will be the content of URL
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth
 -   `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of URL (optional, default `false`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
@@ -126,7 +127,7 @@ Take `Object` as parameters of URL, throw each chunk from the result
 
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch (by default input string is taken)
 -   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose the path to split JSON result (optional, default `*`)
--   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in seconds (optional, default `1000`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 

diff --git a/packages/basics/src/url-connect.js b/packages/basics/src/url-connect.js
@@ -10,11 +10,13 @@ import fetch from 'fetch-with-proxy';
  * @name URLConnect
  * @param {String} [url] URL to fecth
  * @param {String} [json=false] Pasre as JSON the content of URL
- * @param {Number} [timeout=1000] Timeout in seconds
+ * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Boolean} [noerror=false] Ignore all errors
  * @returns {Object}
  */
 export default function URLConnect(data, feed) {
     const url = this.getParam('url');
+    const noerror = Boolean(this.getParam('noerror', false));
     const json = this.getParam('json', true);
     const { ezs } = this;
     if (this.isFirst()) {
@@ -30,8 +32,7 @@ export default function URLConnect(data, feed) {
             .then(({ body, status, statusText }) => {
                 if (status !== 200) {
                     const msg = `Received status code ${status} (${statusText})`;
-                    this.whenFinish = Promise.resolve(true);
-                    return Promise.reject(new Error(msg));
+                    throw new Error(msg);
                 }
                 const output = json ? body.pipe(JSONStream.parse('*')) : body;
                 output.once('error', () => controller.abort());
@@ -40,7 +41,11 @@ export default function URLConnect(data, feed) {
             })
             .catch((e) => {
                 controller.abort();
-                feed.stop(e);
+                this.whenFinish = Promise.resolve(true);
+                if (!noerror) {
+                    feed.stop(e);
+                }
+                return Promise.resolve(true);
             });
     }
     if (this.isLast()) {

diff --git a/packages/basics/src/url-fetch.js b/packages/basics/src/url-fetch.js
@@ -10,7 +10,7 @@ import fetch from 'fetch-with-proxy';
  * @param {String} [url] URL to fecth
  * @param {String} [target] choose the key to set
  * @param {String} [json=false] Parse as JSON the content of URL
- * @param {Number} [timeout=1000] Timeout in seconds
+ * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
  * @returns {Object}
  */

diff --git a/packages/basics/src/url-stream.js b/packages/basics/src/url-stream.js
@@ -10,7 +10,7 @@ import fetch from 'fetch-with-proxy';
  * @name URLStream
  * @param {String} [url] URL to fetch (by default input string is taken)
  * @param {String} [path=*] choose the path to split JSON result
- * @param {Number} [timeout=1000] Timeout in seconds
+ * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
  * @returns {Object}
  */

diff --git a/packages/basics/test/url-connect.js b/packages/basics/test/url-connect.js
@@ -74,6 +74,27 @@ describe('URLConnect', () => {
                 done(new Error('Error is the right behavior'));
             });
     });
+    test('#3bis', (done) => {
+        ezs.use(statements);
+        const input = [1, 2, 3, 4, 5];
+        const output = [];
+        from(input)
+            .pipe(ezs('URLConnect', {
+                url: 'http://127.0.0.1:33331/nofound.ini',
+                noerror: true,
+            }))
+            .pipe(ezs.catch())
+            .on('error', () => {
+                done(new Error('Error should be ignored'));
+            })
+            .on('data', (chunk) => {
+                output.push(chunk);
+            })
+            .on('end', () => {
+                expect(output.length).toBe(0);
+                done();
+            });
+    });
     test('#4', (done) => {
         ezs.use(statements);
         const input = ['1a', '2a', '3a', '4a', '5a'];