diff --git a/package.json b/package.json index e9e2f8b..d2fdec3 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "dependencies": { "@comicrelief/lambda-wrapper": "^1.1.10", "axios": "^0.21.0", - "got": "^11.8.0", + "got": "10.2.2", "hpagent": "^0.1.1" } } diff --git a/src/Common/data-processor-got.js b/src/Common/data-processor-got.js index 7b6b8b8..314a51d 100644 --- a/src/Common/data-processor-got.js +++ b/src/Common/data-processor-got.js @@ -1,8 +1,20 @@ import * as got from 'got'; +import { HttpsProxyAgent } from 'hpagent'; +import * as https from 'https'; +import * as http from 'http'; -export default class DataProcessorGot { +const urlToOptions = require('got/dist/source/utils/url-to-options').default; +const instance = got.extend({ + request: (url, options, callback) => { + if(url.protocol === 'https') { + return https.request({...options, ...urlToOptions(url)}, callback); + } + return http.request({...options, ...urlToOptions(url)}, callback); + } +}); - async getWebData() { +export default class DataProcessorGot { + async getWebData() { console.log('Getting data'); let response = await this.getContent('https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY'); return response; @@ -14,13 +26,26 @@ export default class DataProcessorGot { async getContent(url) { try { - const response = await got(url, { JSON: true }); + const response = await instance(url, { + // agent: { + // https: new HttpsProxyAgent({ + // keepAlive: true, + // keepAliveMsecs: 1000, + // maxSockets: 256, + // maxFreeSocket: 256, + // scheduling: 'lifo', + // proxy: 'http://ipv4.86.161.174.79.webdefence.global.blackspider.com:8080' + // }) + // }, + JSON: true, + // followRedirect: false, + }); return JSON.parse(response.body); } catch (error) { console.log('Caught error in DataProcessorGot'); - // console.log(error.options); + console.log(error.options); console.log('Here is the RequestError:'); - //console.log(JSON.parse(error).RequestError); + console.log(error.RequestError); console.log('Here is the full Error object'); console.log(error); throw error; diff --git a/yarn.lock b/yarn.lock index 12fb7a6..5bc96ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1551,10 +1551,10 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== -"@sindresorhus/is@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.0.0.tgz#2ff674e9611b45b528896d820d3d7a812de2f0e4" - integrity sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ== +"@sindresorhus/is@^1.0.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-1.2.0.tgz#63ce3638cb85231f3704164c90a18ef816da3fb7" + integrity sha512-mwhXGkRV5dlvQc4EgPDxDxO6WuMBVymGFd1CA+2Y+z5dG9MNspoQ+AWjl/Ld1MnpCL8AKbosZlDVohqcIwuWsw== "@sinonjs/commons@^1.7.0": version "1.8.1" @@ -1577,7 +1577,7 @@ dependencies: defer-to-connect "^1.0.1" -"@szmarczak/http-timer@^4.0.5": +"@szmarczak/http-timer@^4.0.0": version "4.0.5" resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.5.tgz#bfbd50211e9dfa51ba07da58a14cdfd333205152" integrity sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ== @@ -1695,7 +1695,7 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.1.2.tgz#4929992f87a0129f4960a110faeb526210562e7b" integrity sha512-IiPhNnenzkqdSdQH3ifk9LoX7oQe61ZlDdDO4+MUv6FyWdPGDPr26gCPVs3oguZEMq//nFZZpwUZcVuNJsG+DQ== -"@types/responselike@*", "@types/responselike@^1.0.0": +"@types/responselike@*": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29" integrity sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA== @@ -2608,10 +2608,12 @@ cache-base@^1.0.1: union-value "^1.0.0" unset-value "^1.0.0" -cacheable-lookup@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3" - integrity sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w== +cacheable-lookup@^0.2.1: + version "0.2.2" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-0.2.2.tgz#5db9a480af3c55494e0a8e57c3c5095c3d314dd3" + integrity sha512-+XR+sYSw3gWyyL/4em004eP3RzisWDaGnBPbGpXl4GFSJSiliomxBU9AvStC6W9ivFuDbEcHwuWCEoSqBrK8Ww== + dependencies: + keyv "^4.0.0" cacheable-request@^6.0.0: version "6.1.0" @@ -2626,7 +2628,7 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" -cacheable-request@^7.0.1: +cacheable-request@^7.0.0: version "7.0.1" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.1.tgz#062031c2856232782ed694a257fa35da93942a58" integrity sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw== @@ -3197,12 +3199,12 @@ decompress-response@^3.3.0: dependencies: mimic-response "^1.0.0" -decompress-response@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" - integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== +decompress-response@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-5.0.0.tgz#7849396e80e3d1eba8cb2f75ef4930f76461cb0f" + integrity sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw== dependencies: - mimic-response "^3.1.0" + mimic-response "^2.0.0" deep-extend@^0.6.0: version "0.6.0" @@ -4212,22 +4214,25 @@ google-protobuf@^3.5.0: resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.13.0.tgz#909c5983d75dd6101ed57c79e0528d000cdc3251" integrity sha512-ZIf3qfLFayVrPvAjeKKxO5FRF1/NwRxt6Dko+fWEMuHwHbZx8/fcaAao9b0wCM6kr8qeg2te8XTpyuvKuD9aKw== -got@^11.8.0: - version "11.8.0" - resolved "https://registry.yarnpkg.com/got/-/got-11.8.0.tgz#be0920c3586b07fd94add3b5b27cb28f49e6545f" - integrity sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ== +got@10.2.2: + version "10.2.2" + resolved "https://registry.yarnpkg.com/got/-/got-10.2.2.tgz#3cd61dc07db0533c6190a2889b78e0c39c0f52d6" + integrity sha512-QibZN13xHH/mc7H5uuU2xq28xxs8moEPsJrW9AQQX0jAV4DkGdllHDVE9cxw1nntIPFk8xzzOrgJZBg194AWrg== dependencies: - "@sindresorhus/is" "^4.0.0" - "@szmarczak/http-timer" "^4.0.5" + "@sindresorhus/is" "^1.0.0" + "@szmarczak/http-timer" "^4.0.0" "@types/cacheable-request" "^6.0.1" - "@types/responselike" "^1.0.0" - cacheable-lookup "^5.0.3" - cacheable-request "^7.0.1" - decompress-response "^6.0.0" - http2-wrapper "^1.0.0-beta.5.2" + cacheable-lookup "^0.2.1" + cacheable-request "^7.0.0" + decompress-response "^5.0.0" + duplexer3 "^0.1.4" + get-stream "^5.0.0" lowercase-keys "^2.0.0" + mimic-response "^2.0.0" p-cancelable "^2.0.0" responselike "^2.0.0" + to-readable-stream "^2.0.0" + type-fest "^0.8.0" got@^9.6.0: version "9.6.0" @@ -4389,14 +4394,6 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -http2-wrapper@^1.0.0-beta.5.2: - version "1.0.0-beta.5.2" - resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.0-beta.5.2.tgz#8b923deb90144aea65cf834b016a340fc98556f3" - integrity sha512-xYz9goEyBnC8XwXDTuC/MZ6t+MrKVQZOk4s7+PaDkwIsQd8IwqvM+0M6bA/2lvG8GHXcPdf+MejTUeO2LCPCeQ== - dependencies: - quick-lru "^5.1.1" - resolve-alpn "^1.0.0" - https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" @@ -5863,10 +5860,10 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== -mimic-response@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" - integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== +mimic-response@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43" + integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA== minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1: version "1.0.1" @@ -6732,11 +6729,6 @@ querystring@0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= -quick-lru@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" - integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== - randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -7007,11 +6999,6 @@ reserved-words@^0.1.2: resolved "https://registry.yarnpkg.com/reserved-words/-/reserved-words-0.1.2.tgz#00a0940f98cd501aeaaac316411d9adc52b31ab1" integrity sha1-AKCUD5jNUBrqqsMWQR2a3FKzGrE= -resolve-alpn@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c" - integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA== - resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -7845,6 +7832,11 @@ to-readable-stream@^1.0.0: resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q== +to-readable-stream@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-2.1.0.tgz#82880316121bea662cdc226adb30addb50cb06e8" + integrity sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w== + to-regex-range@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" @@ -7971,7 +7963,7 @@ type-fest@^0.6.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== -type-fest@^0.8.1: +type-fest@^0.8.0, type-fest@^0.8.1: version "0.8.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==