Skip to content

Commit

Permalink
Added urlToOptions suggested at https://runkit.com/szmarczak/5e1a23d8…
Browse files Browse the repository at this point in the history 
…e4d193001a709110 from sindresorhus/got#876 still not working
  • Loading branch information
corinja committed Nov 4, 2020
1 parent 8a87492 commit 7908d41
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 56 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"dependencies": {
"@comicrelief/lambda-wrapper": "^1.1.10",
"axios": "^0.21.0",
"got": "^11.8.0",
"got": "10.2.2",
"hpagent": "^0.1.1"
}
}
35 changes: 30 additions & 5 deletions src/Common/data-processor-got.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,20 @@
import * as got from 'got';
import { HttpsProxyAgent } from 'hpagent';
import * as https from 'https';
import * as http from 'http';

export default class DataProcessorGot {
const urlToOptions = require('got/dist/source/utils/url-to-options').default;
const instance = got.extend({
request: (url, options, callback) => {
if(url.protocol === 'https') {
return https.request({...options, ...urlToOptions(url)}, callback);
}
return http.request({...options, ...urlToOptions(url)}, callback);
}
});

async getWebData() {
export default class DataProcessorGot {
async getWebData() {
console.log('Getting data');
let response = await this.getContent('https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY');
return response;
Expand All @@ -14,13 +26,26 @@ export default class DataProcessorGot {

async getContent(url) {
try {
const response = await got(url, { JSON: true });
const response = await instance(url, {
// agent: {
// https: new HttpsProxyAgent({
// keepAlive: true,
// keepAliveMsecs: 1000,
// maxSockets: 256,
// maxFreeSocket: 256,
// scheduling: 'lifo',
// proxy: 'http://ipv4.86.161.174.79.webdefence.global.blackspider.com:8080'
// })
// },
JSON: true,
// followRedirect: false,
});
return JSON.parse(response.body);
} catch (error) {
console.log('Caught error in DataProcessorGot');
// console.log(error.options);
console.log(error.options);
console.log('Here is the RequestError:');
//console.log(JSON.parse(error).RequestError);
console.log(error.RequestError);
console.log('Here is the full Error object');
console.log(error);
throw error;
Expand Down
92 changes: 42 additions & 50 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1551,10 +1551,10 @@
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==

"@sindresorhus/is@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.0.0.tgz#2ff674e9611b45b528896d820d3d7a812de2f0e4"
integrity sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ==
"@sindresorhus/is@^1.0.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-1.2.0.tgz#63ce3638cb85231f3704164c90a18ef816da3fb7"
integrity sha512-mwhXGkRV5dlvQc4EgPDxDxO6WuMBVymGFd1CA+2Y+z5dG9MNspoQ+AWjl/Ld1MnpCL8AKbosZlDVohqcIwuWsw==

"@sinonjs/commons@^1.7.0":
version "1.8.1"
Expand All @@ -1577,7 +1577,7 @@
dependencies:
defer-to-connect "^1.0.1"

"@szmarczak/http-timer@^4.0.5":
"@szmarczak/http-timer@^4.0.0":
version "4.0.5"
resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.5.tgz#bfbd50211e9dfa51ba07da58a14cdfd333205152"
integrity sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==
Expand Down Expand Up @@ -1695,7 +1695,7 @@
resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.1.2.tgz#4929992f87a0129f4960a110faeb526210562e7b"
integrity sha512-IiPhNnenzkqdSdQH3ifk9LoX7oQe61ZlDdDO4+MUv6FyWdPGDPr26gCPVs3oguZEMq//nFZZpwUZcVuNJsG+DQ==

"@types/responselike@*", "@types/responselike@^1.0.0":
"@types/responselike@*":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29"
integrity sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==
Expand Down Expand Up @@ -2608,10 +2608,12 @@ cache-base@^1.0.1:
union-value "^1.0.0"
unset-value "^1.0.0"

cacheable-lookup@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3"
integrity sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w==
cacheable-lookup@^0.2.1:
version "0.2.2"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-0.2.2.tgz#5db9a480af3c55494e0a8e57c3c5095c3d314dd3"
integrity sha512-+XR+sYSw3gWyyL/4em004eP3RzisWDaGnBPbGpXl4GFSJSiliomxBU9AvStC6W9ivFuDbEcHwuWCEoSqBrK8Ww==
dependencies:
keyv "^4.0.0"

cacheable-request@^6.0.0:
version "6.1.0"
Expand All @@ -2626,7 +2628,7 @@ cacheable-request@^6.0.0:
normalize-url "^4.1.0"
responselike "^1.0.2"

cacheable-request@^7.0.1:
cacheable-request@^7.0.0:
version "7.0.1"
resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.1.tgz#062031c2856232782ed694a257fa35da93942a58"
integrity sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==
Expand Down Expand Up @@ -3197,12 +3199,12 @@ decompress-response@^3.3.0:
dependencies:
mimic-response "^1.0.0"

decompress-response@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc"
integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==
decompress-response@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-5.0.0.tgz#7849396e80e3d1eba8cb2f75ef4930f76461cb0f"
integrity sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw==
dependencies:
mimic-response "^3.1.0"
mimic-response "^2.0.0"

deep-extend@^0.6.0:
version "0.6.0"
Expand Down Expand Up @@ -4212,22 +4214,25 @@ google-protobuf@^3.5.0:
resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.13.0.tgz#909c5983d75dd6101ed57c79e0528d000cdc3251"
integrity sha512-ZIf3qfLFayVrPvAjeKKxO5FRF1/NwRxt6Dko+fWEMuHwHbZx8/fcaAao9b0wCM6kr8qeg2te8XTpyuvKuD9aKw==

got@^11.8.0:
version "11.8.0"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.0.tgz#be0920c3586b07fd94add3b5b27cb28f49e6545f"
integrity sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ==
got@10.2.2:
version "10.2.2"
resolved "https://registry.yarnpkg.com/got/-/got-10.2.2.tgz#3cd61dc07db0533c6190a2889b78e0c39c0f52d6"
integrity sha512-QibZN13xHH/mc7H5uuU2xq28xxs8moEPsJrW9AQQX0jAV4DkGdllHDVE9cxw1nntIPFk8xzzOrgJZBg194AWrg==
dependencies:
"@sindresorhus/is" "^4.0.0"
"@szmarczak/http-timer" "^4.0.5"
"@sindresorhus/is" "^1.0.0"
"@szmarczak/http-timer" "^4.0.0"
"@types/cacheable-request" "^6.0.1"
"@types/responselike" "^1.0.0"
cacheable-lookup "^5.0.3"
cacheable-request "^7.0.1"
decompress-response "^6.0.0"
http2-wrapper "^1.0.0-beta.5.2"
cacheable-lookup "^0.2.1"
cacheable-request "^7.0.0"
decompress-response "^5.0.0"
duplexer3 "^0.1.4"
get-stream "^5.0.0"
lowercase-keys "^2.0.0"
mimic-response "^2.0.0"
p-cancelable "^2.0.0"
responselike "^2.0.0"
to-readable-stream "^2.0.0"
type-fest "^0.8.0"

got@^9.6.0:
version "9.6.0"
Expand Down Expand Up @@ -4389,14 +4394,6 @@ http-signature@~1.2.0:
jsprim "^1.2.2"
sshpk "^1.7.0"

http2-wrapper@^1.0.0-beta.5.2:
version "1.0.0-beta.5.2"
resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.0-beta.5.2.tgz#8b923deb90144aea65cf834b016a340fc98556f3"
integrity sha512-xYz9goEyBnC8XwXDTuC/MZ6t+MrKVQZOk4s7+PaDkwIsQd8IwqvM+0M6bA/2lvG8GHXcPdf+MejTUeO2LCPCeQ==
dependencies:
quick-lru "^5.1.1"
resolve-alpn "^1.0.0"

https-browserify@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"
Expand Down Expand Up @@ -5863,10 +5860,10 @@ mimic-response@^1.0.0, mimic-response@^1.0.1:
resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==

mimic-response@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==
mimic-response@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==

minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1:
version "1.0.1"
Expand Down Expand Up @@ -6732,11 +6729,6 @@ querystring@0.2.0:
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=

quick-lru@^5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932"
integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==

randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
Expand Down Expand Up @@ -7007,11 +6999,6 @@ reserved-words@^0.1.2:
resolved "https://registry.yarnpkg.com/reserved-words/-/reserved-words-0.1.2.tgz#00a0940f98cd501aeaaac316411d9adc52b31ab1"
integrity sha1-AKCUD5jNUBrqqsMWQR2a3FKzGrE=

resolve-alpn@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c"
integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA==

resolve-cwd@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d"
Expand Down Expand Up @@ -7845,6 +7832,11 @@ to-readable-stream@^1.0.0:
resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771"
integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==

to-readable-stream@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-2.1.0.tgz#82880316121bea662cdc226adb30addb50cb06e8"
integrity sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w==

to-regex-range@^2.1.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38"
Expand Down Expand Up @@ -7971,7 +7963,7 @@ type-fest@^0.6.0:
resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b"
integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==

type-fest@^0.8.1:
type-fest@^0.8.0, type-fest@^0.8.1:
version "0.8.1"
resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
Expand Down

0 comments on commit 7908d41

Please sign in to comment.