diff --git a/package-lock.json b/package-lock.json index e53e84d..22edd35 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "audible-library-extractor", - "version": "0.2.8", + "version": "0.2.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "audible-library-extractor", - "version": "0.2.8", + "version": "0.2.9", "dependencies": { "@fortawesome/fontawesome-svg-core": "^1.2.35", "@fortawesome/free-brands-svg-icons": "^5.15.3", @@ -16,15 +16,15 @@ "@growthbunker/vuedarkmode": "^0.5.56", "@splidejs/vue-splide": "^0.3.5", "@vueform/multiselect": "^2.0.1", - "async-es": "^3.2.0", - "axios": "^0.21.4", + "async-es": "^3.2.3", + "axios": "^0.26.1", "axios-rate-limit": "^1.3.0", - "axios-retry": "^3.1.9", + "axios-retry": "^3.2.4", "buefy": "^0.9.8", "bulma": "^0.9.3", "clean-webpack-plugin": "^3.0.0", "date-fns": "^2.22.1", - "dompurify": "^2.3.0", + "dompurify": "^2.3.6", "domurl": "^2.3.4", "downloadjs": "^1.4.7", "eruda": "^2.4.1", @@ -37,6 +37,7 @@ "jszip": "^3.7.1", "jszip-utils": "^0.1.0", "lodash": "^4.17.21", + "normalize.css": "^8.0.1", "one-colorpicker": "^1.0.12", "retry-axios": "^2.4.0", "smoothscroll-polyfill": "^0.4.4", @@ -1570,10 +1571,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.14.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.14.6.tgz", - "integrity": "sha512-/PCB2uJ7oM44tz8YhC4Z/6PeOKXp4K588f+5M3clr1M4zbqztlo0XEfJ2LEzj/FgwfgGcIdl8n7YYjTCI0BYwg==", - "dev": true, + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", "dependencies": { "regenerator-runtime": "^0.13.4" }, @@ -2438,9 +2438,9 @@ "dev": true }, "node_modules/async-es": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async-es/-/async-es-3.2.0.tgz", - "integrity": "sha512-dMWVIUBi/ejt+QERtMdIin2rlzSNK7GAfEwIyaOGDy536s0OqH+aUNkj9MQ01hq2rB1f7x6w7RJchtthCk0IDA==" + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async-es/-/async-es-3.2.3.tgz", + "integrity": "sha512-Wk2Lu2UmwScooActfrWSqcsYtlLt8Kl21d8ly3A5kbbXgRN8pVy4cCotXnoBAiKGjuE2XqSUw+0uj9b7x18aoQ==" }, "node_modules/async-foreach": { "version": "0.1.3", @@ -2473,11 +2473,11 @@ "dev": true }, "node_modules/axios": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", "dependencies": { - "follow-redirects": "^1.14.0" + "follow-redirects": "^1.14.8" } }, "node_modules/axios-rate-limit": { @@ -2489,11 +2489,12 @@ } }, "node_modules/axios-retry": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.1.9.tgz", - "integrity": "sha512-NFCoNIHq8lYkJa6ku4m+V1837TP6lCa7n79Iuf8/AqATAHYB0ISaAS1eyIenDOfHOLtym34W65Sjke2xjg2fsA==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.2.4.tgz", + "integrity": "sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ==", "dependencies": { - "is-retry-allowed": "^1.1.0" + "@babel/runtime": "^7.15.4", + "is-retry-allowed": "^2.2.0" } }, "node_modules/babel-loader": { @@ -2790,13 +2791,19 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001243", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001243.tgz", - "integrity": "sha512-vNxw9mkTBtkmLFnJRv/2rhs1yufpDfCkBZexG3Y0xdOH2Z/eE/85E4Dl5j1YUN34nZVsSp6vVRFQRrez9wJMRA==", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - } + "version": "1.0.30001327", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001327.tgz", + "integrity": "sha512-1/Cg4jlD9qjZzhbzkzEaAC2JHsP0WrOc8Rd/3a3LuajGzGWR/hD7TVyvq99VqmTy99eVh8Zkmdq213OgvgXx7w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] }, "node_modules/caseless": { "version": "0.12.0", @@ -3592,9 +3599,9 @@ } }, "node_modules/dompurify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.0.tgz", - "integrity": "sha512-VV5C6Kr53YVHGOBKO/F86OYX6/iLTw2yVSI721gKetxpHCK/V5TaLEf9ODjRgl1KLSWRMY6cUhAbv/c+IUnwQw==" + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.6.tgz", + "integrity": "sha512-OFP2u/3T1R5CEgWCEONuJ1a5+MFKnOYpkywpUSxv/dj1LeBT1erK+JwM7zK0ROy2BRhqVCf0LRw/kHqKuMkVGg==" }, "node_modules/domurl": { "version": "2.3.4", @@ -4848,11 +4855,14 @@ } }, "node_modules/is-retry-allowed": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", - "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", "engines": { - "node": ">=0.10.0" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/is-stream": { @@ -5802,6 +5812,11 @@ "node": ">=0.10.0" } }, + "node_modules/normalize.css": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/normalize.css/-/normalize.css-8.0.1.tgz", + "integrity": "sha512-qizSNPO93t1YUuUhP22btGOo3chcvDFqFaj2TRybP0DMxkHOCTYwp3n34fel4a31ORXy4m1Xq0Gyqpb5m33qIg==" + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6615,8 +6630,7 @@ "node_modules/regenerator-runtime": { "version": "0.13.7", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==", - "dev": true + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" }, "node_modules/regenerator-transform": { "version": "0.14.5", @@ -9494,10 +9508,9 @@ } }, "@babel/runtime": { - "version": "7.14.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.14.6.tgz", - "integrity": "sha512-/PCB2uJ7oM44tz8YhC4Z/6PeOKXp4K588f+5M3clr1M4zbqztlo0XEfJ2LEzj/FgwfgGcIdl8n7YYjTCI0BYwg==", - "dev": true, + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", "requires": { "regenerator-runtime": "^0.13.4" } @@ -10235,9 +10248,9 @@ "dev": true }, "async-es": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async-es/-/async-es-3.2.0.tgz", - "integrity": "sha512-dMWVIUBi/ejt+QERtMdIin2rlzSNK7GAfEwIyaOGDy536s0OqH+aUNkj9MQ01hq2rB1f7x6w7RJchtthCk0IDA==" + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async-es/-/async-es-3.2.3.tgz", + "integrity": "sha512-Wk2Lu2UmwScooActfrWSqcsYtlLt8Kl21d8ly3A5kbbXgRN8pVy4cCotXnoBAiKGjuE2XqSUw+0uj9b7x18aoQ==" }, "async-foreach": { "version": "0.1.3", @@ -10264,11 +10277,11 @@ "dev": true }, "axios": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", "requires": { - "follow-redirects": "^1.14.0" + "follow-redirects": "^1.14.8" } }, "axios-rate-limit": { @@ -10278,11 +10291,12 @@ "requires": {} }, "axios-retry": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.1.9.tgz", - "integrity": "sha512-NFCoNIHq8lYkJa6ku4m+V1837TP6lCa7n79Iuf8/AqATAHYB0ISaAS1eyIenDOfHOLtym34W65Sjke2xjg2fsA==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.2.4.tgz", + "integrity": "sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ==", "requires": { - "is-retry-allowed": "^1.1.0" + "@babel/runtime": "^7.15.4", + "is-retry-allowed": "^2.2.0" } }, "babel-loader": { @@ -10496,9 +10510,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001243", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001243.tgz", - "integrity": "sha512-vNxw9mkTBtkmLFnJRv/2rhs1yufpDfCkBZexG3Y0xdOH2Z/eE/85E4Dl5j1YUN34nZVsSp6vVRFQRrez9wJMRA==" + "version": "1.0.30001327", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001327.tgz", + "integrity": "sha512-1/Cg4jlD9qjZzhbzkzEaAC2JHsP0WrOc8Rd/3a3LuajGzGWR/hD7TVyvq99VqmTy99eVh8Zkmdq213OgvgXx7w==" }, "caseless": { "version": "0.12.0", @@ -11102,9 +11116,9 @@ } }, "dompurify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.0.tgz", - "integrity": "sha512-VV5C6Kr53YVHGOBKO/F86OYX6/iLTw2yVSI721gKetxpHCK/V5TaLEf9ODjRgl1KLSWRMY6cUhAbv/c+IUnwQw==" + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.6.tgz", + "integrity": "sha512-OFP2u/3T1R5CEgWCEONuJ1a5+MFKnOYpkywpUSxv/dj1LeBT1erK+JwM7zK0ROy2BRhqVCf0LRw/kHqKuMkVGg==" }, "domurl": { "version": "2.3.4", @@ -12075,9 +12089,9 @@ } }, "is-retry-allowed": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", - "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==" }, "is-stream": { "version": "2.0.0", @@ -12820,6 +12834,11 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "normalize.css": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/normalize.css/-/normalize.css-8.0.1.tgz", + "integrity": "sha512-qizSNPO93t1YUuUhP22btGOo3chcvDFqFaj2TRybP0DMxkHOCTYwp3n34fel4a31ORXy4m1Xq0Gyqpb5m33qIg==" + }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -13430,8 +13449,7 @@ "regenerator-runtime": { "version": "0.13.7", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==", - "dev": true + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" }, "regenerator-transform": { "version": "0.14.5", diff --git a/package.json b/package.json index 9242134..b87e12f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "audible-library-extractor", - "version": "0.2.8", + "version": "0.2.9", "description": "Extracts your Audible library metadata and automatically generates a locally viewable gallery that can also be uploaded online.", "author": "Joonas Pääkkö ", "engines": { @@ -23,15 +23,15 @@ "@growthbunker/vuedarkmode": "^0.5.56", "@splidejs/vue-splide": "^0.3.5", "@vueform/multiselect": "^2.0.1", - "async-es": "^3.2.0", - "axios": "^0.21.4", + "async-es": "^3.2.3", + "axios": "^0.26.1", "axios-rate-limit": "^1.3.0", - "axios-retry": "^3.1.9", + "axios-retry": "^3.2.4", "buefy": "^0.9.8", "bulma": "^0.9.3", "clean-webpack-plugin": "^3.0.0", "date-fns": "^2.22.1", - "dompurify": "^2.3.0", + "dompurify": "^2.3.6", "domurl": "^2.3.4", "downloadjs": "^1.4.7", "eruda": "^2.4.1", @@ -44,6 +44,7 @@ "jszip": "^3.7.1", "jszip-utils": "^0.1.0", "lodash": "^4.17.21", + "normalize.css": "^8.0.1", "one-colorpicker": "^1.0.12", "retry-axios": "^2.4.0", "smoothscroll-polyfill": "^0.4.4", @@ -68,6 +69,7 @@ "@babel/preset-env": "^7.14.7", "@babel/runtime-corejs3": "^7.14.7", "@vue/composition-api": "^1.0.0-rc.13", + "ansi-regex": ">=5.0.1", "archiver": "^5.3.0", "babel-loader": "^8.2.2", "chunks-2-json-webpack-plugin": "^1.0.4", @@ -87,7 +89,6 @@ "webpack": "^5.43.0", "webpack-bundle-analyzer": "^4.4.2", "webpack-cli": "^4.7.2", - "webpack-extension-reloader": "^1.1.4", - "ansi-regex": ">=5.0.1" + "webpack-extension-reloader": "^1.1.4" } } diff --git a/src/content-script/_components/_mixins/calls/amapxios.js b/src/content-script/_components/_mixins/calls/amapxios.js index 26ba915..3021520 100644 --- a/src/content-script/_components/_mixins/calls/amapxios.js +++ b/src/content-script/_components/_mixins/calls/amapxios.js @@ -3,54 +3,88 @@ import rateLimit from "axios-rate-limit"; export default { methods: { + + minutesToMilliseconds: function( minutes ) { return minutes * 60 * 1000; }, + secondsToMilliseconds: function( seconds ) { return seconds * 1000; }, + amapxios: function(options) { - const vue = this; - const letMeAxiosAQuestion = axios.create(); - axiosRetry(letMeAxiosAQuestion, { - retries: 2, + const limiter = options.rateLimit || 50; + const timeout = this.minutesToMilliseconds(1); + + // AXIOS + let cAxios = axios.create(); + // AXIOS RETRY + axiosRetry(cAxios, { + retries: 1, retryDelay: function(retryCount) { return 1000 * retryCount; }, retryCondition: function(error) { - return axiosRetry.isNetworkOrIdempotentRequestError(error) || error && error.response && ( error.response.status == "500" || error.response.status == "400" ); + return axiosRetry.isNetworkOrIdempotentRequestError(error) || error && error.response && ( error.response.status == "500" || error.response.status == "400" || error.response.status == "429" ); } }); - const axiosLimited = rateLimit(letMeAxiosAQuestion, { maxRPS: 10 }); + // AXIOS RATE LIMIT + cAxios = rateLimit(cAxios, { maxRPS: limiter }); - asyncMap( + // REQUEST LOOP + asyncMapLimit( options.requests, + limiter, // ASYNC MAP LIMITER function(request, stepCallback) { + const axiosConfig = options.config || {}; + const requestURL = request.requestUrl || request.url || request; + const controller = new AbortController(); + + axiosConfig.signal = controller.signal; + axiosConfig.validateStatus = function (status) { + return status >= 200 && status < 400; + }; + + let requestTimer = setTimeout(() => { + controller.abort('Request took too long... ('+ requestURL +')'); + }, timeout); - axiosLimited - .get((request.requestUrl || request.url || request), axiosConfig) - .then(function(response) { - options.step( response,function(result) { - stepCallback(null, result); - }, request ); - }) - .catch(function(e) { - console.log( "%c" + "axios caught an error (step)" + "", "background: #f41b1b; color: #fff; padding: 2px 5px; border-radius: 8px;", e ); - if ( options.returnCatch ) { - options.step( e.response, function(result) { - stepCallback(null, result); - }, request, 'processingError' ); - } - else { - stepCallback(null, null); - } + cAxios.get(requestURL, axiosConfig).then(function(response) { + + clearTimeout(requestTimer); // Request succeeded: keep alive + + options.step( response,function(result) { + stepCallback(null, result); + }, request ); + + }) + .catch(function(e) { + + clearTimeout(requestTimer); // Request succeeded: keep alive + + console.log( "%c" + "axios caught an error (step)" + "", "background: #f41b1b; color: #fff; padding: 2px 5px; border-radius: 8px;", '\n', requestURL, '\n', e ); + if ( options.returnCatch ) { + options.step( e.response, function(result) { + stepCallback(null, result); + }, request, 'processingError' ); + } + else { + stepCallback(null, null); + } + }); + }, function(err, results) { + if (!err) { results = options.flatten ? _.flatten(results) : results; results = _.compact( results ); options.done( results ); - } else { + } + else { console.log( "%c" + "axios caught an error (done)" + "", "background: #f79a33; color: #fff; padding: 2px 5px; border-radius: 8px;", err ); - options.done( null ); + options.done( null, true ); } + } ); - } + + }, } }; diff --git a/src/content-script/_components/_mixins/calls/scrapingPrep.js b/src/content-script/_components/_mixins/calls/scrapingPrep.js index 2449e9d..3563a96 100644 --- a/src/content-script/_components/_mixins/calls/scrapingPrep.js +++ b/src/content-script/_components/_mixins/calls/scrapingPrep.js @@ -7,10 +7,12 @@ import rateLimit from "axios-rate-limit"; export default { methods: { - scrapingPrep: function( baseUrl, callbach, returnResponse, returnAfterFirstCall, maxSize ) { - const vue = this; - + + scrapingPrep: function( config ) { + const vue = this; + config = config || {}; + const letMeAxiosAQuestion = axios.create(); axiosRetry(letMeAxiosAQuestion, { retries: 2, @@ -19,35 +21,30 @@ export default { return axiosRetry.isNetworkOrIdempotentRequestError(error) || error && error.response && error.response.status == "500"; } }); - const axiosLimited = rateLimit(letMeAxiosAQuestion, { maxRPS: 10 }); + const axiosLimited = rateLimit(letMeAxiosAQuestion, { maxRPS: 15 }); waterfall( [ + // GET MAX PAGE SIZE (how many items per page) function(callback) { - let url = new Url( DOMPurify.sanitize(baseUrl) ); + let url = new Url( DOMPurify.sanitize(config.url) ); url.query.ale = true; url.query.bp_ua = 'yes'; - - axiosLimited.get(url.toString()).then(function(response) { - const audible = $($.parseHTML(response.data)).find("div.adbl-main"); - const pageSizeDropdown = audible.find('select[name="pageSize"]'); - const maxPageSize = pageSizeDropdown.length > 0 ? DOMPurify.sanitize(pageSizeDropdown.find("option:last").val()) : null; - url.query.pageSize = maxSize ? (maxPageSize > maxSize ? maxSize : maxPageSize) : maxPageSize; - - let obj = {}; - if (returnResponse) obj.response = response; - obj.urlObj = url; - - if (!maxPageSize || maxPageSize < 50 || returnAfterFirstCall) { - obj.pageNumbers = [1]; - obj.pageSize = maxPageSize; - callback(true, obj); - } else { - callback(null, obj); - } - }); + let obj = {}; + obj.urlObj = url; + + if ( config.skipFirstCall ) { + vue.getMaxPageSize( obj, config, config.response, callback); + } + else { + axiosLimited.get(url.toString()).then(function(response) { + vue.getMaxPageSize( obj, config, response, callback); + }); + } + }, + // GET MAX PAGE COUNT (pagination) function(o, callback) { axiosLimited.get(o.urlObj.toString()).then(function(response) { const audible = $($.parseHTML(response.data)).find("div.adbl-main"); @@ -60,9 +57,39 @@ export default { } ], function(err, obj) { - callbach(obj); + config.done(obj); } ); - } + }, + + getMaxPageSize: function( obj, config, response, waterfallback ) { + + const audible = $($.parseHTML(response.data)).find("div.adbl-main"); + const pageSizeDropdown = audible.find('select[name="pageSize"]'); + const maxPageSize = pageSizeDropdown.length > 0 ? DOMPurify.sanitize(pageSizeDropdown.find("option:last").val()) : null; + obj.urlObj.query.pageSize = config.maxSize || obj.urlObj.query.pageSize || maxPageSize; + + if (config.returnResponse) obj.response = response; + + const pagination = audible.find(".pagingElements").length; + if ( !pagination || !maxPageSize || maxPageSize < 50 || config.returnAfterFirstCall ) { + obj.pageNumbers = [1]; + obj.pageSize = obj.urlObj.query.pageSize; + waterfallback(true, obj); // true makes the waterfall jump to the end. + } else { + waterfallback(null, obj); + } + + }, + + getPageNumbers: function( response ) { + + const audible = $($.parseHTML(response.data)).find("div.adbl-main"); + const pagination = audible.find(".pagingElements"); + const pagesLength = pagination.length > 0 ? parseFloat( DOMPurify.sanitize(pagination.find(".pageNumberElement:last").text()) ) : 1; + return _.range(1, pagesLength + 1); + + }, + } }; diff --git a/src/content-script/_components/_mixins/main-step/process-collections.js b/src/content-script/_components/_mixins/main-step/process-collections.js index 24d62b7..93d454e 100644 --- a/src/content-script/_components/_mixins/main-step/process-collections.js +++ b/src/content-script/_components/_mixins/main-step/process-collections.js @@ -4,126 +4,336 @@ export default { getDataFromCollections: function(hotpotato, collectionsFetched) { if ( !_.find(hotpotato.config.steps, { name: "collections" }) ) { - this.$root.$emit("reset-progress"); + this.$store.commit('resetProgress'); + // this.$root.$emit("reset-progress"); collectionsFetched(null, hotpotato); } else { - this.$root.$emit("update-big-step", { - title: "Collections", - stepAdd: 1 - }); + + this.$store.commit('update', [ + { key: 'bigStep.title', value: 'Collections' }, + { key: 'bigStep.step', add: 1 }, + { key: 'progress.text', value: 'Fetching collections...' }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + { key: 'progress.bar', value: true }, + ]); + + // this.$root.$emit("update-big-step", { + // title: "Collections", + // stepAdd: 1 + // }); + // this.$root.$emit("update-progress", { + // text: "Fetching collections...", + // step: 0, + // max: 0, + // bar: true + // }); const vue = this; waterfall( [ - function(callback) { - vue.scrapingPrep(vue.collectionsUrl, function(prep) { - callback(null, prep); - }); - }, // returns {pageNumbers, urlObj} - - // I wouldn't expect the collections page (audible.com/library/collections) to have more than - // 1 page with the highest page size (50), but just in case this functions is ready to make multiple calls... - function(prep, callback) { - vue.$root.$emit("update-progress", { - text: "Fetching collections...", - step: 0, - max: 0, - bar: true - }); - - vue.amapxios({ - requests: _.map(prep.pageNumbers, function(page) { - prep.urlObj.query.page = page; - return prep.urlObj.toString(); - }), - step: function(response, stepCallback) { - getInitialCollectionDataFromPage(vue, response, stepCallback); - }, - flatten: true, - done: function(collections) { - callback(null, collections); - } - }); + + function( callback ) { + vue.uniteBooksInCollections( hotpotato, callback ) }, + + vue.fetchCollectionDetails, + + + // // https://www.audible.com/library/collections + // // Find out the max items per page and max pages + // // Returns _something_ along these lines: + // // { + // // maxSize: 50, + // // pages: 4, + // // } + // vue.getFirstLevelCollectionPages, - // Adds maximum page size and pages to each collection - function(collections, callback) { - asyncMap( - collections, - function(collection, stepCallback) { - vue.scrapingPrep( - vue.collectionsUrl + "/" + collection.id, - function(prep) { - collection.pageNumbers = prep.pageNumbers; - collection.pageSize = prep.pageSize; - collection.url = prep.urlObj.toString(); - getBooks(vue, collection, stepCallback); - } - ); - }, - function(err, responses) { - if (!err) { - let flatResponses = _.flatten(responses); - callback(null, flatResponses, collections); - } else console.log(err); - } - ); - } + // // Returns a list of all pages: + // // https://www.audible.com/library/collections?pageSize=50&page=1 + // // https://www.audible.com/library/collections?pageSize=50&page=2 + // // https://www.audible.com/library/collections?pageSize=50&page=3 + // // https://www.audible.com/library/collections?pageSize=50&page=4 + // vue.getFirstLevelCollectionData, + + // // Adds maximum page size and pages to each collection + // // Note: doesn't scrape these pages, just uses information on the first page of + // // each collection to figure out how many pages they have. + // // Returns something like: + // // https://www.audible.com/library/collections/__FAVORITES?pageSize=50&page1 + // // https://www.audible.com/library/collections/__FAVORITES?pageSize=50&page2 + // // https://www.audible.com/library/collections/dWwhq9p4EWg4QlhbOzCK?pageSize=50&page1 + // // https://www.audible.com/library/collections/dWwhq9p4EWg4QlhbOzCK?pageSize=50&page2 + // // https://www.audible.com/library/collections/dWwhq9p4EWg4QlhbOzCK?pageSize=50&page3 + // // https://www.audible.com/library/collections/dWwhq9p4EWg4QlhbOzCK?pageSize=50&page4 + // // https://www.audible.com/library/collections/259b-ZMcADLvn-It6RE-XuB?pageSize=20&page1 + // vue.getAllCollectionBookPages, + ], - function(err, responses, collections) { + function(err, responses ) { // Pushes books back to the original array of collections without any duplicate ids - _.each(responses, function(collection) { - const targetCollection = _.find(collections, { id: collection.id }); - if ( targetCollection ) { - targetCollection.books = targetCollection.books.concat( collection.books ); - delete targetCollection.pageNumbers; - delete targetCollection.pageSize; - delete targetCollection.url; - } - }); + // _.each(responses, function(collection) { + // const targetCollection = _.find(collections, { id: collection.id }); + // if ( targetCollection ) { + // targetCollection.books = targetCollection.books.concat( collection.books ); + // delete targetCollection.pageNumbers; + // delete targetCollection.pageSize; + // delete targetCollection.url; + // } + // }); - hotpotato.collections = collections; + hotpotato.collections = responses; + vue.$store.commit('resetProgress'); vue.$nextTick(function() { - vue.$root.$emit("reset-progress"); + // vue.$root.$emit("reset-progress"); collectionsFetched(null, hotpotato); }); } ); } - } - } -}; - -function getInitialCollectionDataFromPage(vue, response, completeStep) { - const audible = $($.parseHTML(response.data)).find("div.adbl-main")[0]; - response.data = null; - const collections = []; - const collectionRows = audible.querySelectorAll("#adbl-lib-col-main > .adbl-library-collection-row"); - - $(collectionRows).each(function() { - const _thisRow = this; - - // Ignores empty collections - Assumes it's empty if it doesn't have any product images - if ( _thisRow.querySelector(".product-image-grid-container:not(.empty-product-image-grid)") ) { - let collection = {}; + }, + + /** + * At this point what the extraction has is a bunch of books with an array of collection ids... + * So this function creates a new array with the collectionIds and adds books under those collections + * @param {object} hotpotato ... + * @param {function} waterfallback ... + */ + uniteBooksInCollections: function( hotpotato, waterfallback ) { + + const booksInCollections = _.filter( hotpotato.books, 'collectionIds' ); + const collections = []; + const vue = this; + + _.each(booksInCollections ,function( book ) { + _.each( book.collectionIds, function( collectionId ) { + + let collection = _.find( collections, { id: collectionId }); + + // Create new collection object if it doesn't exist (no duplicates) + if ( !collection ) { + collection = { + id: collectionId, + url: vue.collectionsUrl + '/' + collectionId, // Removed after title and description have been fetched + books: [], + }; + collections.push( collection ); + // vue.$root.$emit("update-progress-max"); + vue.$store.commit('update', { key: 'progress.max', add: 1 }); + } + + // Add book to the collection object + if ( collection ) collection.books.push( book.asin ); + + + }); + }); + + waterfallback( null, collections ); + + }, + + fetchCollectionDetails: function( collections, waterfallback ) { + + const vue = this; + this.amapxios({ + requests: collections, + step: function(response, stepCallback, request) { + + const audible = $($.parseHTML(response.data)).find("div.adbl-main")[0]; + + // TITLE + let title = audible.querySelector('#center-3 > div.bc-container > div > div > span > ul > li:nth-child(1) > span.bc-text.bc-size-headline2'); + if ( title ) title = title.textContent; + if ( title ) request.title = title; + + // DESCRIPTION + let description = audible.querySelector('#center-3 > div.bc-container > div > div > span > ul > li.bc-list-item.summaryLabel.bc-spacing-s0_5 > span'); + if ( description ) description = description.textContent; + if ( description ) request.description = description; + + // vue.$root.$emit("update-progress-step"); + vue.$store.commit('update', { key: 'progress.step', add: 1 }); + + stepCallback( request ); + + }, + flatten: true, + done: function(collections) { + + _.each( collections, function( collection ) { + if ( collection.url ) delete collection.url; + }); + + waterfallback(null, collections ); + + } + }); + + + }, + + getFirstLevelCollectionData: function(requests, callback) { + + const vue = this; + + // vue.$root.$emit("update-progress", { + // text: "Fetching collections...", + // step: 0, + // max: 0, + // bar: true + // }); + + vue.$store.commit('update', [ + { key: 'progress.text', value: 'Fetching collections...' }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + { key: 'progress.bar', value: true }, + ]); + + vue.amapxios({ + requests: requests, + step: function(response, stepCallback) { + vue.scrapeFirstLevelCollectionData(response, stepCallback); + }, + flatten: true, + done: function(collections) { + callback(null, collections); + } + }); + }, + + /** + * + * @param {obj} response ... + * @param {function} completeStep callback + */ + scrapeFirstLevelCollectionData: function(response, completeStep) { + + const vue = this; + const audible = $($.parseHTML(response.data)).find("div.adbl-main")[0]; + response.data = null; + const collections = []; + const collectionRows = audible.querySelectorAll("#adbl-lib-col-main > .adbl-library-collection-row"); - collection.id = DOMPurify.sanitize( _thisRow.getAttribute("data-collection-id") ); // Collection page is formed using the id: audible.com/library/collections/{id} - collection.title = DOMPurify.sanitize( _thisRow.querySelector(".bc-size-headline3").textContent ); - const description = _thisRow.querySelector("ul .bc-text.bc-size-body.bc-color-secondary"); - if (description) collection.description = DOMPurify.sanitize( description.textContent ); - collection.books = []; + each( collectionRows, function( _thisRow ) { - collections.push(collection); - vue.$root.$emit("update-progress-max"); - } - }); + // If preview grid container is empty, ignore the collection + const previewGridHasItems = _thisRow.querySelector(".product-image-grid-container:not(.empty-product-image-grid)"); + if ( previewGridHasItems ) { + + let collection = {}; + + // ID + const id =_thisRow.getAttribute("data-collection-id"); + collection.id = DOMPurify.sanitize( id ); // Collection page is formed using the id: audible.com/library/collections/{id} + // TITLE + const title = _thisRow.querySelector(".bc-size-headline3") + if ( title ) collection.title = DOMPurify.sanitize( title.textContent ); + // DESCRIPTION + const description = _thisRow.querySelector("ul .bc-text.bc-size-body.bc-color-secondary"); + if ( description ) collection.description = DOMPurify.sanitize( description.textContent ); + collection.books = []; + + collections.push(collection); + + } + + // vue.$root.$emit("update-progress-max"); + vue.$store.commit('update', { key: 'progress.max', add: 1 }); + + }); - completeStep(collections); -} + completeStep(collections); + + }, + + getAllCollectionBookPages: function( collections, waterfallback ) { + + const vue = this; + _.each(collections, function(collection) { + collection.url = vue.collectionsUrl + "/" + collection.id + '?pageSize=50'; + }); + + vue.amapxios({ + requests: collections, + rateLimit: 20, + step: function(response, stepCallback, request) { + + vue.scrapingPrep({ + url: response.request.responseURL || request.url, + skipFirstCall: true, + response: response, + done: function(prep) { + + // vue.$root.$emit("update-progress-max"); + vue.$store.commit('update', { key: 'progress.max', add: 1 }); + + request.pageNumbers = prep.pageNumbers; + request.pageSize = prep.pageSize; + request.url = prep.urlObj.toString(); + + stepCallback(request); + + } + }); + + }, + flatten: true, + done: function( responses, err ) { + + let requestUrls = []; + + if (!err) { + + _.each( responses, function( request ) { + _.each(request.pageNumbers, function(page) { + const requestDolly = JSON.parse(JSON.stringify(request)); + let url = new Url( DOMPurify.sanitize(requestDolly.url) ); + url.query.page = page; + url.query.pageSize = request.pageSize; + requestDolly.url = url.toString(); + requestUrls.push(requestDolly); + }); + }); + + } else console.log(err); + + waterfallback(null, requestUrls, responses); + + } + }); + + }, + + /** + * Fetches all pages for the first level of collections + * @param {function} callback + */ + getFirstLevelCollectionPages: function( callback ) { + + this.scrapingPrep({ + url: this.collectionsUrl, + done: function(prep) { + + const requestURLS = _.map(prep.pageNumbers, function(page) { + prep.urlObj.query.page = page; + return prep.urlObj.toString(); + }); + + callback(null, requestURLS); + + } + }); + + }, + + } +}; function getBooks(vue, request, parentStepCallback) { let requestUrls = []; @@ -161,7 +371,8 @@ function getBooks(vue, request, parentStepCallback) { }, flatten: true, done: function(collections) { - vue.$root.$emit("update-progress-step"); // Counting collections, not books + // vue.$root.$emit("update-progress-step"); // Counting collections, not books + vue.$store.commit('update', { key: 'progress.step', add: 1 }); parentStepCallback(null, collections); } }); diff --git a/src/content-script/_components/_mixins/main-step/process-isbns.js b/src/content-script/_components/_mixins/main-step/process-isbns.js index 98596f1..e5d016f 100644 --- a/src/content-script/_components/_mixins/main-step/process-isbns.js +++ b/src/content-script/_components/_mixins/main-step/process-isbns.js @@ -6,24 +6,34 @@ export default { getISBNsFromGoogleBooks: function(hotpotato, isbnsFetched) { if ( !_.find(hotpotato.config.steps, { name: "isbn" }) ) { - this.$root.$emit("reset-progress"); + // this.$root.$emit("reset-progress"); + this.$store.commit("resetProgress"); isbnsFetched(null, hotpotato); } else { - this.$root.$emit("update-big-step", { - title: "International Standard Book Number (ISBN)", - stepAdd: 1 - }); + // this.$root.$emit("update-big-step", { + // title: "International Standard Book Number (ISBN)", + // stepAdd: 1 + // }); - this.$root.$emit("update-progress", { - text2: - "(The matching process is relatively loose: beware of false matches)", - text: "Fetching ISBNs from Google Books API...", - step: 0, - max: 0, - bar: true - }); + // this.$root.$emit("update-progress", { + // text2: "(The matching process is relatively loose: beware of false matches)", + // text: "Fetching ISBNs from Google Books API...", + // step: 0, + // max: 0, + // bar: true + // }); + + this.$store.commit('update', [ + { key: 'bigStep.title', value: 'International Standard Book Number (ISBN)' }, + { key: 'bigStep.step', add: 1 }, + { key: 'progress.text2', value: "(The matching process is relatively loose: beware of false matches)" }, + { key: 'progress.text', value: "Fetching ISBNs from Google Books API..." }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + { key: 'progress.bar', value: true }, + ]); const vue = this; fetchISBNs(vue, hotpotato, isbnsFetched); @@ -54,7 +64,9 @@ function fetchISBNs(vue, hotpotato, isbnsFetched) { } }); - vue.$root.$emit("update-progress", { max: requestUrls.length }); + // vue.$root.$emit("update-progress", { max: requestUrls.length }); + vue.$store.commit('update', { key: 'progress.max', add: requestUrls.length }); + const letMeAxiosAQuestion = axios.create(); axiosRetry(letMeAxiosAQuestion, { retries: 7, @@ -71,17 +83,22 @@ function fetchISBNs(vue, hotpotato, isbnsFetched) { perMilliseconds: 1100 }); - asyncMap( + asyncMapLimit( requestUrls, + 18, function(request, stepCallback) { isbn - .get(request.url) + .get(request.url, { + validateStatus: function (status) { + return status >= 200 && status < 400; + } + }) .then(function(response) { const book = _.find(hotpotato.books, { asin: request.asin }); if ( response && response.status >= 200 && - response.status < 300 && + response.status < 400 && response.data && response.data.totalItems ) { @@ -115,9 +132,11 @@ function fetchISBNs(vue, hotpotato, isbnsFetched) { book.isbns = isbns; if (_.get(api_book, "volumeInfo.imageLinks.smallThumbnail")) { - vue.$root.$emit( - "update-progress-thumbnail", DOMPurify.sanitize( api_book.volumeInfo.imageLinks.smallThumbnail.replace("http://","https://") ) - ); + // vue.$root.$emit( + // "update-progress-thumbnail", DOMPurify.sanitize( api_book.volumeInfo.imageLinks.smallThumbnail.replace("http://","https://") ) + // ); + const thumbnail = DOMPurify.sanitize( api_book.volumeInfo.imageLinks.smallThumbnail.replace("http://","https://") ); + vue.$store.commit('update', { key: 'progress.thumbnail', value: thumbnail }); } } @@ -132,7 +151,8 @@ function fetchISBNs(vue, hotpotato, isbnsFetched) { ); } - vue.$root.$emit("update-progress-step"); + // vue.$root.$emit("update-progress-step"); + vue.$store.commit('update', { key: 'progress.step', add: 1 }); stepCallback(null); }) .catch(e => { @@ -141,14 +161,16 @@ function fetchISBNs(vue, hotpotato, isbnsFetched) { "background: #f41b1b; color: #fff; padding: 2px 5px; border-radius: 8px;", e.respone ); - vue.$root.$emit("update-progress-step"); + // vue.$root.$emit("update-progress-step"); + vue.$store.commit('update', { key: 'progress.step', add: 1 }); stepCallback(null); }); }, function(err, result) { if (!err) { vue.$nextTick(function() { - vue.$root.$emit("reset-progress"); + // vue.$root.$emit("reset-progress"); + vue.$store.commit('resetProgress'); isbnsFetched(null, hotpotato); }); } else console.log(err); diff --git a/src/content-script/_components/_mixins/main-step/process-library-pages-fin.js b/src/content-script/_components/_mixins/main-step/process-library-pages-fin.js index ce8c442..6738019 100644 --- a/src/content-script/_components/_mixins/main-step/process-library-pages-fin.js +++ b/src/content-script/_components/_mixins/main-step/process-library-pages-fin.js @@ -9,21 +9,24 @@ export default { text: 'Double checking book status...' }); - vue.scrapingPrep(vue.libraryUrlFin, function(prep) { + vue.scrapingPrep({ + url: vue.libraryUrlFin, + done: function(prep) { - const requestURL = prep.urlObj.toString(); - vue.amapxios({ - requests: _.map(prep.pageNumbers, function(page) { - return requestURL + "&page=" + page; - }), - step: function(response, stepCallback) { - processLibraryPage(vue, response, hotpotato, stepCallback); - }, - flatten: true, - done: function() { - libraryPagesFetched(null, hotpotato); - } - }); + const requestURL = prep.urlObj.toString(); + vue.amapxios({ + requests: _.map(prep.pageNumbers, function(page) { + return requestURL + "&page=" + page; + }), + step: function(response, stepCallback) { + processLibraryPage(vue, response, hotpotato, stepCallback); + }, + flatten: true, + done: function() { + libraryPagesFetched(null, hotpotato); + } + }); + } }); } @@ -42,7 +45,7 @@ function processLibraryPage(vue, response, hotpotato, stepCallback) { response.data = null; const titleRows = audible.querySelectorAll("#adbl-library-content-main > .adbl-library-content-row"); - titleRows.forEach(function(el) { + each( titleRows, function(el) { const _thisRow = el; const rowItemIsBook = diff --git a/src/content-script/_components/_mixins/main-step/process-library-pages.js b/src/content-script/_components/_mixins/main-step/process-library-pages.js index 94113af..02f4028 100644 --- a/src/content-script/_components/_mixins/main-step/process-library-pages.js +++ b/src/content-script/_components/_mixins/main-step/process-library-pages.js @@ -7,68 +7,84 @@ export default { if ( _.find(hotpotato.config.steps, { name: "library" }) ) { - this.$root.$emit("update-big-step", { - title: "Library", - stepAdd: 1 - }); - - this.$root.$emit("update-progress", { - step: 0, - max: 0, - text: this.storageHasData.books ? "Updating old books and adding new books..." : "Scanning library for books..." - }); + this.$store.commit('update', [ + { key: 'bigStep.title', value: 'Library' }, + { key: 'bigStep.step', add: 1 }, + { key: 'subStep.step', add: 1 }, + { key: 'subStep.max', value: 5 }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + { key: 'progress.text', value: this.storageHasData.books ? "Updating old books and adding new books..." : "Scanning library for books..." }, + ]); - vue.scrapingPrep(vue.libraryUrl, function(prep) { - - const requestURL = prep.urlObj.toString(); - vue.amapxios({ - requests: _.map(prep.pageNumbers, function(page) { - return requestURL + "&page=" + page; // + '&stampyStamp=' + (new Date().getTime()); - }), - step: function(response, stepCallback) { - vue.processLibraryPage(response, hotpotato, stepCallback); - }, - flatten: true, - done: function(books) { - vue.$nextTick(function() { - - hotpotato.books = books; - - // Removes unnecessary data from series and collections durin a partial extraction - if ( hotpotato.books && hotpotato.books.length ) { - const changedBooks = _.xorBy( hotpotato.books, books, 'asin'); - if ( changedBooks.length > 0 ) { - let removedBooks = _.filter( changedBooks, function( book ) { return !book.isNewThisRound; }); - if ( removedBooks.length > 0 ) { - vue.removeFromSeries( hotpotato.series, removedBooks ); - vue.removeFromCollections( hotpotato.collections, removedBooks ); + vue.scrapingPrep({ + url: vue.libraryUrl, + maxSize: 20, + done: function(prep) { + + const requestURL = prep.urlObj.toString(); + vue.amapxios({ + requests: _.map(prep.pageNumbers, function(page) { + return requestURL + "&page=" + page; // + '&stampyStamp=' + (new Date().getTime()); + }), + step: function(response, stepCallback) { + vue.processLibraryPage(response, hotpotato, stepCallback); + }, + flatten: true, + done: function(books) { + vue.$nextTick(function() { + + hotpotato.books = books; + + // Removes unnecessary data from series and collections durin a partial extraction + if ( hotpotato.books && hotpotato.books.length ) { + const changedBooks = _.xorBy( hotpotato.books, books, 'asin'); + if ( changedBooks.length > 0 ) { + let removedBooks = _.filter( changedBooks, function( book ) { return !book.isNewThisRound; }); + if ( removedBooks.length > 0 ) { + vue.removeFromSeries( hotpotato.series, removedBooks ); + vue.removeFromCollections( hotpotato.collections, removedBooks ); + } } - } - if ( hotpotato.wishlist && hotpotato.wishlist.length > 0 ) { - let newBooks = _.filter( hotpotato.books, 'isNewThisRound'); - if ( newBooks.length > 0 ) { - newBooks = _.map(newBooks, 'asin'); - _.remove( hotpotato.wishlist, function( wBook ) { - return _.includes(newBooks, wBook.asin); - }); + if ( hotpotato.wishlist && hotpotato.wishlist.length > 0 ) { + let newBooks = _.filter( hotpotato.books, 'isNewThisRound'); + if ( newBooks.length > 0 ) { + newBooks = _.map(newBooks, 'asin'); + _.remove( hotpotato.wishlist, function( wBook ) { + return _.includes(newBooks, wBook.asin); + }); + } } } - } - - // hotpotato.books = _.filter( hotpotato.books, { asin: 'B078X15P2P' }); - - hotpotato.config.getStorePages = 'books'; - libraryPagesFetched(null, hotpotato); - }); - } - }); - }, false, false, 20); + // hotpotato.books = _.sampleSize( hotpotato.books, 40); + + // hotpotato.books = _.sampleSize( hotpotato.books, function( book ) { + + // return _.includes([ + // 'B078X15P2P', + // '1799751511', + // '1781103798', + // ], book.asin); + + // }); + + hotpotato.config.getStorePages = 'books'; + vue.$nextTick(function() { + libraryPagesFetched(null, hotpotato); + }); + + }); + } + }); + + } + }); } else { - vue.$root.$emit("reset-progress"); + vue.$store.commit('resetProgress'); libraryPagesFetched(null, hotpotato); } @@ -85,9 +101,7 @@ export default { const books = []; const titleRows = audible.querySelectorAll("#adbl-library-content-main > .adbl-library-content-row"); - titleRows.forEach(function(el) { - const _thisRow = el; - + each( titleRows, function( _thisRow ) { // const rowItem = { // is: { @@ -146,7 +160,7 @@ export default { // "Came from the plus catalog but is no longer available there" const unavailableBtn = _thisRow.querySelector(".adbl-library-inaccessible-button"); if (unavailableBtn) { book.unavailable = true; } - else { delete book.unavailable; } + else if ( book.unavailable ) { delete book.unavailable; } // Downloaded book.downloaded = _thisRow.querySelector(".adbl-library-action > div:nth-child(4) > span") ? true : null; @@ -154,6 +168,7 @@ export default { // Favorite const favorite = _thisRow.querySelector('[id^="remove-from-favorites-button"]:not(.bc-pub-hidden)'); if (favorite) book.favorite = true; + else if ( book.favorite ) delete book.favorite; // Progress const progressbar = _thisRow.querySelector('[id^="time-remaining-display"] [role="progressbar"]'); @@ -167,6 +182,16 @@ export default { book.progress = 0; } + // Collection IDs + let collectionIds = _thisRow.querySelector('[id^="collectionIds-"]'); + if ( book.collectionIds ) delete book.collectionIds; + if ( collectionIds ) collectionIds = collectionIds.getAttribute('value'); + if ( collectionIds ) collectionIds = collectionIds.replace(/^\[/, '').replace(/\]$/, '').trim(); + if ( collectionIds ) collectionIds = collectionIds.split(', '); + if ( collectionIds && collectionIds.length > 0 ) { + _.remove( collectionIds, function( value ) { return value === '__PENDING' || value.trim() === ''; }); + if ( collectionIds.length > 0 ) book.collectionIds = collectionIds; + } // if ( // book.title.lastIndexOf('Besiege') > -1 || @@ -226,7 +251,7 @@ export default { if (fullScan_ALL_partialScan_NEW) { book.isNewThisRound = true; - vue.$root.$emit("update-progress-max"); + vue.$store.commit('update', { key: 'progress.max', add: 1 }); } books.push(book); diff --git a/src/content-script/_components/_mixins/main-step/process-purchase-history.js b/src/content-script/_components/_mixins/main-step/process-purchase-history.js index daa3244..42d6c22 100644 --- a/src/content-script/_components/_mixins/main-step/process-purchase-history.js +++ b/src/content-script/_components/_mixins/main-step/process-purchase-history.js @@ -18,12 +18,16 @@ export default { text: this.storageHasData.books ? "Updating old books and adding new books..." : "Scanning library for books..." }); - vue.scrapingPrep(vue.purchaseHistoryUrl, function(prep) { + vue.scrapingPrep({ + url: vue.purchaseHistoryUrl, + maxSize: 20, + done: function(prep) { - const requestURL = prep.urlObj.toString(); - console.log( prep ) - - }, false, false, 20); + const requestURL = prep.urlObj.toString(); + console.log( prep ) + + } + }); }, diff --git a/src/content-script/_components/_mixins/main-step/process-series-pages.js b/src/content-script/_components/_mixins/main-step/process-series-pages.js index 2212350..e669288 100644 --- a/src/content-script/_components/_mixins/main-step/process-series-pages.js +++ b/src/content-script/_components/_mixins/main-step/process-series-pages.js @@ -1,25 +1,54 @@ export default { methods: { + + filterBooksInSeries: function( hotpotato ) { + + let booksInSeries = []; + + const config = _.get(hotpotato, 'config'); + const step = { + library: _.find(_.get(config, 'steps'), { name: "library" }), + test: _.get(config, 'seriesTest'), + }; + + const books = _.get(hotpotato, 'books', []); + + if ( step.library ) booksInSeries = _.filter(books, function(o) { return o.isNewThisRound && o.series; }); + else if ( step.test ) booksInSeries = _.filter(books, function(o) { return o.series; }); + + return booksInSeries; + + }, + getDataFromSeriesPages: function(hotpotato, seriesFetched) { const vue = this; - let booksInSeries = []; - if ( _.find(hotpotato.config.steps, { name: "library" }) || hotpotato.config.seriesTest ) { - booksInSeries = vue.storageHasData.books ? _.filter(hotpotato.books, 'isNewThisRound') : hotpotato.books; - } + // FILTER SERIES + let booksInSeries = this.filterBooksInSeries(hotpotato); + + // NO SERIES: move on to next step if ( !booksInSeries.length ) { + this.$store.commit('update', { key: 'subStep.step', add: 3 }); moveOn(); } + // BOOKS IN SERIES: start extracting information about the series else { - booksInSeries = _.filter(booksInSeries, "series"); - let requests = []; + // Update progress UI + this.$store.commit('update', [ + { key: 'progress.text', value: "Compiling a list of series pages..." }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + { key: 'progress.bar', value: true }, + ]); + // PREP REQUESTS + let requests = []; _.each(booksInSeries, function(book) { _.each(book.series, function(series) { requests.push({ - url: vue.seriesUrl + "/" + series.asin, + url: vue.seriesUrl + "/" + series.asin + '?pageSize=50', asin: series.asin, books: [], allBooks: [], @@ -27,33 +56,33 @@ export default { }); }); }); - requests = _.uniqBy(requests, "asin"); - this.$root.$emit("update-progress", { - text: "Preparing books in series...", - step: 0, - max: 0, - bar: true - }); - - asyncMap( - requests, - function(request, stepCallback) { - vue.scrapingPrep(request.url, function(prep) { - vue.$root.$emit("update-progress", { max: requests.length }); - - request.pageNumbers = prep.pageNumbers; - request.pageSize = prep.pageSize; - vue.getBooksFromSeries(hotpotato, request, stepCallback); - - }); - }, + // START MAKING CALLS + waterfall( + [ + // Fetches request urls for every single page of every single series + function(waterfallback) { + vue.getAllSeriesPages( requests, waterfallback) + }, + // Scrapes each page for every single series page + function(requests, waterfallback) { + vue.getBooksFromSeries( hotpotato, requests, waterfallback ); + }, + // Last effort to try to find missing book numbers from store pages + function(requests, waterfallback) { + vue.getMissingNumbers( hotpotato, requests, waterfallback ); + } + ], function(err, responses) { + resetProgress(); + + // FIXME: everything in this function is a bit janky. Could just push into hotpotato.series within each step...? + // Each series page is fetched as a separate request. // This merges the books arrays and cleans up the returning object a little - _.each(_.flatten(responses), function(series) { + _.each(responses, function(series) { const targetSeries = _.find(requests, { asin: series.asin }); if (targetSeries) { targetSeries.books = targetSeries.books.concat(series.books); @@ -65,9 +94,16 @@ export default { } }); - if ( vue.storageHasData.books && _.get(hotpotato, 'series') ) { - vue.mergeBooksWithSeries( hotpotato.series, requests ); + // IF SERIES HAVE BEEN EXTRACTED, MERGE FETCHED SERIES WITH THOSE + const potatoSeries = _.get(hotpotato, 'series', []); + if ( vue.storageHasData.books && potatoSeries.length ) { + _.each( requests, function( series ) { + const seriesExists = _.find(potatoSeries, { asin: series.asin }); + if ( seriesExists ) _.merge( seriesExists, series ); + else hotpotato.series.push( series ); + }); } + // ALL NEW SERIES: dump requests into potato as is else { hotpotato.series = requests; } @@ -81,10 +117,20 @@ export default { } + function resetProgress() { + + vue.$store.commit('update', [ + { key: 'subStep.step', value: 0 }, + { key: 'subStep.max', value: 0 }, + ]); + vue.$store.commit('resetProgress'); + + } + function moveOn() { + resetProgress(); vue.$nextTick(function() { - vue.$root.$emit("reset-progress"); seriesFetched(null, hotpotato); }); @@ -92,201 +138,259 @@ export default { }, - fetchMissingNumbers: function( missingNumbers, series, parentStepCallback ) { + getAllSeriesPages: function( requests, waterfallback ) { + + this.$store.commit("update", { key: 'subStep.step', add: 1 }); - let vue = this; + const vue = this; vue.amapxios({ - requests: _.map( missingNumbers, function( book ) { - return { - requestUrl: window.location.origin +'/pd/'+ book.asin, - seriesAsin: series.asin, - bookAsin: book.asin, - }; - }), + requests: requests, + limiter: 100, step: function(response, stepCallback, request) { - let seriesBook = {}; - - if (response.status < 400) { - let html = $($.parseHTML(response.data)); - let audible = html.find("div.adbl-main")[0]; - html = null; - let bookSeries = vue.getSeries(audible.querySelector(".seriesLabel")); - bookSeries = _.find(bookSeries, { asin: request.seriesAsin }); - if ( bookSeries && bookSeries.bookNumbers ) { - seriesBook = { - asin: request.bookAsin, - bookNumbers: bookSeries.bookNumbers.join(',') - }; - } - } + request.pageNumbers = vue.getPageNumbers( response ); + request.pageSize = '50'; - stepCallback(seriesBook); + vue.$store.commit("update", { key: 'progress.max', add: 1 }); + stepCallback(request); }, flatten: true, - done: function( newBooks ) { + done: function( responses, err ) { - _.each( newBooks, function( newBook ) { - let targetBook = _.find( series.allBooks, { asin: newBook.asin }); - if ( targetBook ) { - targetBook.bookNumbers = newBook.bookNumbers; - } + let requestUrls = []; + _.each( responses, function( request ) { + _.each(request.pageNumbers, function( page ) { + const requestDolly = JSON.parse(JSON.stringify(request)); + let url = new Url( DOMPurify.sanitize(requestDolly.url) ); + url.query.page = page; + url.query.pageSize = request.pageSize; + requestDolly.url = url.toString(); + requestUrls.push(requestDolly); + }); }); - parentStepCallback(series); + waterfallback(null, requestUrls); + } }); }, - getBooksFromSeries(hotpotato, request, parentStepCallback) { - let vue = this; - let requestUrls = []; - - _.each(request.pageNumbers, function(page) { - const requestDolly = _.cloneDeep(request); - const pageSize = request.pageSize ? "&pageSize=" + request.pageSize : ""; - requestDolly.url += "/?page=" + page + pageSize; - requestUrls.push(requestDolly); - }); - + getBooksFromSeries(hotpotato, requestUrls, parentStepCallback) { + + const vue = this; + + this.$store.commit('update', [ + { key: 'progress.text', value: 'Fetching series order...' }, + { key: 'progress.max', value: requestUrls.length }, + { key: 'subStep.step', add: 1 }, + ]); + vue.amapxios({ requests: requestUrls, step: function(response, stepCallback, request) { const audible = $($.parseHTML(response.data)).find("div.adbl-main")[0]; response.data = null; + let series = null; const bookRows = audible.querySelectorAll(".adbl-impression-container > .productListItem"); - - let series = { - asin: request.asin, - books: [], - allBooks: [], - length: bookRows.length - }; - - let prevBook = {}; - - $(bookRows).each(function() { - - let inLibrary; - const asinEl = this.querySelector("div[data-asin]"); - let titleShort = DOMPurify.sanitize(this.getAttribute('aria-label')); - - let title; - let subtitle = this.querySelector('.subtitle'); - if ( subtitle ) { - subtitle = DOMPurify.sanitize( subtitle.textContent.trimAll() ); - title = titleShort + ': ' + subtitle; - } - // If book has no subtitle, the only existing title must be the long title... - else { - title = titleShort; - titleShort = false; - } + if ( bookRows.length > 0 ) { - if ( title === titleShort ) { - titleShort = false; - } + series = { + asin: request.asin, + books: [], + allBooks: [], + length: bookRows.length + }; - if ( asinEl ) { - const asin = asinEl.getAttribute("data-asin"); - if ( this.querySelector(".adblBuyBoxInLibraryButton") ) { - series.books.push( DOMPurify.sanitize(asin) ); - } + each( bookRows, function( row ) { - } - // Sometimes books may leave the store or it is blocked in your region or something. - // This makes it so the book you have doesn't match a book in the series page. - // So what I'm doing is matching the title to a title in the library so it is then - // possible to push the book asin in the series collection at the right location. - else { + let inLibrary; + const asinEl = row.querySelector("div[data-asin]"); + let titleShort = DOMPurify.sanitize(row.getAttribute('aria-label')); - if ( title ) { - inLibrary = _.find( hotpotato.books, { 'title': title }); + let title; + let subtitle = row.querySelector('.subtitle'); + if ( subtitle ) { + subtitle = DOMPurify.sanitize( subtitle.textContent.trimAll() ); + title = titleShort + ': ' + subtitle; } - - // TitleShort Fallback... - // Title short is not as accurate for matching purposes, but it's better than nothing... - if ( !inLibrary ) { - inLibrary = _.find( hotpotato.books, { 'titleShort': titleShort }); + // If book has no subtitle, the only existing title must be the long title... + else { + title = titleShort; + titleShort = false; } - if ( inLibrary ) series.books.push( inLibrary.asin ); + if ( title === titleShort ) titleShort = false; - } - - let aBook = {}; - - if ( title ) aBook.title = title; - if ( titleShort ) aBook.titleShort = titleShort; - let numbers = this.querySelector(':scope > div:nth-child(1) > div > h2'); - if ( numbers ) aBook.bookNumbers = DOMPurify.sanitize( numbers.textContent.trimAll().replace(/[^\d]*/, '').split(',')[0] ); - if ( asinEl ) aBook.asin = DOMPurify.sanitize( asinEl.getAttribute("data-asin") ); - else if ( inLibrary ) aBook.asin = inLibrary.asin; - - if ( !inLibrary && !this.querySelector('.adblBuyBoxInLibraryButton') ) { - aBook.notInLibrary = true; - } - - // Try to get book number from the title if that fails, - // mark it as numberless with the infinity symbol... - if ( !aBook.bookNumbers ) { - let findBookNumber = aBook.title.match(/(?:, Book.)(.+)/); - if ( aBook.title && findBookNumber ) { - aBook.bookNumbers = findBookNumber[1].replace(/\)$/, ''); + if ( asinEl ) { + const asin = asinEl.getAttribute("data-asin"); + if ( row.querySelector(".adblBuyBoxInLibraryButton") ) { + series.books.push( DOMPurify.sanitize(asin) ); + } } + // Sometimes books may leave the store or it is blocked in your region or something. + // This makes it so the book you have doesn't match a book in the series page. + // So what I'm doing is matching the title to a title in the library so it is then + // possible to push the book asin in the series collection at the right location. else { - aBook.bookNumbers = (function( books, seriesAsin, bookAsin ) { + const potatoBooks = _.get( hotpotato, 'books', []); + // Try to match the title to an existing book in the library + if ( title ) inLibrary = _.find( potatoBooks, { 'title': title }); + // TitleShort Fallback... + // Title short is not as accurate for matching purposes, but it's better than nothing... + if ( !inLibrary ) inLibrary = _.find( potatoBooks, { 'titleShort': titleShort }); + // The book is in the library so push it into series + if ( inLibrary ) series.books.push( inLibrary.asin ); + } + + // FORM BOOK OBJECTS WITH THE COLLECTED DATA + let aBook = {}; + // Book ASIN + if ( asinEl ) aBook.asin = DOMPurify.sanitize( asinEl.getAttribute("data-asin") ); + else if ( inLibrary ) aBook.asin = inLibrary.asin; // Just in case library asin is different... + // Book title + if ( title ) aBook.title = title; + if ( titleShort ) aBook.titleShort = titleShort; + // Book number as displayed in the series page... if at all + let numbers = row.querySelector(':scope > div:nth-child(1) > div > h2'); + if ( numbers ) numbers = numbers.textContent; + if ( numbers ) aBook.bookNumbers = DOMPurify.sanitize( numbers.trimAll().replace(/[^\d]*/, '').split(',')[0] ); + // NOT in library + if ( !inLibrary && !row.querySelector('.adblBuyBoxInLibraryButton') ) { + aBook.notInLibrary = true; + } + // In plus catalog + if ( row.querySelector('[name="discovery-add-to-library-form"]') ) aBook.plus = true; + // It's a free book... + if ( row.querySelector('.buybox-regular-price') ) { + let price = row.querySelector('.buybox-regular-price').textContent.match(/\d/g); + let priceArray = _.map( price, function( v ) { return parseFloat(v) }); + let sumOfPriceArray = priceArray.reduce((a, b) => a + b, 0); + if ( sumOfPriceArray === 0 ) aBook.free = true; + } + + // Try to get book number from the title if that fails, + // mark it as numberless with the infinity symbol... + if ( !aBook.bookNumbers ) { + let findBookNumber; + if ( aBook.title ) findBookNumber = aBook.title.match(/(?:, Book.)(.+)/); + if ( findBookNumber ) findBookNumber = findBookNumber[1]; + if ( findBookNumber ) aBook.bookNumbers = findBookNumber.replace(/\)$/, ''); + } + // Try to get book numbers from existing books in the library using ASIN. + if ( !aBook.bookNumbers ) { + aBook.bookNumbers = (function( books, seriesAsin, bookAsin ) { + const book = _.find(books, { asin: bookAsin }); if ( !book ) return; const series = _.find(book.series, { asin: seriesAsin }); if ( !series ) return; - return series.bookNumbers.join(','); + return _.isArray(series.bookNumbers) ? series.bookNumbers.join(',') : series.bookNumbers; })( hotpotato.books, request.asin, aBook.asin ); - } - } - - if ( !aBook.bookNumbers ) aBook.bookNumbers = '∞'; - - if ( this.querySelector('[name="discovery-add-to-library-form"]') ) aBook.plus = true; - // if ( title.match(/^FREE:/) ) aBook.free = true; - if ( this.querySelector('.buybox-regular-price') ) { - let price = this.querySelector('.buybox-regular-price').textContent.match(/\d/g); - let priceArray = _.map( price, function( v ) { return parseFloat(v) }); - let sumOfPriceArray = priceArray.reduce((a, b) => a + b, 0); - if ( sumOfPriceArray === 0 ) aBook.free = true; - } - - series.allBooks.push( aBook ); + // IF a book doesn't have a number, make it the infinity symbol... + if ( !aBook.bookNumbers ) aBook.bookNumbers = '∞'; + + series.allBooks.push( aBook ); + + }); - }); - - vue.$root.$emit("update-progress", { - text: "Fetching series order for books in series..." - }); + } + vue.$store.commit('update', { key: 'progress.step', add: 1 }); + stepCallback(series); - // Final attempt at finding missing numbers by checking series pages: - let missingNumbers = _.filter( series.allBooks, { bookNumbers: '∞' }); - if ( missingNumbers ) { - vue.fetchMissingNumbers( missingNumbers, series, stepCallback ); - } - else { - stepCallback(series); + }, + flatten: true, + done: function(series) { + parentStepCallback(null, series); + } + }); + }, + + // Final attempt at finding missing numbers by checking book's store pages + getMissingNumbers: function( hotpotato, series, waterfallback ) { + + // Prep requests + let booksWithMissingNumber = []; + _.each( series, function( currentSeries ) { + let missing = _.filter( currentSeries.allBooks, { bookNumbers: '∞' }); + if ( missing.length ) { + missing = _.map( missing, function( book ) { + return { + requestUrl: window.location.origin +'/pd/'+ book.asin, + seriesAsin: currentSeries.asin, + bookAsin: book.asin, + }; + }) + booksWithMissingNumber = booksWithMissingNumber.concat(missing); + } + }); + + // Update progress GUI + this.$store.commit('update', [ + { key: 'progress.text', value: 'Attempting to fetch missing numbers from store pages...' }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: booksWithMissingNumber.length }, + { key: 'progress.bar', value: true }, + { key: 'subStep.step', add: 1 }, + ]); + + let vue = this; + vue.amapxios({ + requests: booksWithMissingNumber, + returnCatch: true, + limiter: 100, + step: function(response, stepCallback, request) { + + let seriesBook = null; + const responseStatus = _.get( response, 'status', 0); + if ( responseStatus >= 200 && responseStatus < 400 ) { + let html = $($.parseHTML(response.data)); + let audible = html.find("div.adbl-main")[0]; + html = null; + if ( audible && audible.querySelector('[id^="sample-player-"] > button') ) { + + const serieslEl = audible.querySelector(".seriesLabel"); + let bookSeries = vue.getSeries( serieslEl ); + if ( bookSeries ) bookSeries = _.find(bookSeries, { asin: request.seriesAsin }); + const bookNumbers = _.get( bookSeries, 'bookNumbers' ); + if ( bookNumbers ) { + seriesBook = { + seriesAsin: request.seriesAsin, + asin: request.bookAsin, + bookNumbers: _.isArray(bookNumbers) ? bookNumbers.join(',') : bookNumbers, + }; + } + + } } + vue.$store.commit('update', { key: 'progress.step', add: 1 }); + stepCallback(seriesBook); + }, flatten: true, - done: function(series) { - vue.$root.$emit("update-progress-step"); // Counting series, not books - parentStepCallback(null, series); + done: function( newBooks ) { + + newBooks = _.omitBy(newBooks, _.isNull); + + _.each( newBooks, function( newBook ) { + let targetSeries = _.find( series, { asin: newBook.seriesAsin }); + if ( targetSeries ) { + let targetBook = _.find( targetSeries.allBooks, { asin: newBook.asin }); + if ( targetBook ) targetBook.bookNumbers = newBook.bookNumbers; + } + }); + + waterfallback(null, series); } }); + }, - + }, }; diff --git a/src/content-script/_components/_mixins/main-step/process-store-pages.js b/src/content-script/_components/_mixins/main-step/process-store-pages.js index 01c9838..e12d73c 100644 --- a/src/content-script/_components/_mixins/main-step/process-store-pages.js +++ b/src/content-script/_components/_mixins/main-step/process-store-pages.js @@ -2,8 +2,11 @@ export default { methods: { getDataFromStorePages: function(hotpotato, storePagesFetched) { + this.$store.commit('update', { key: 'subStep.step', add: 1 }); + const vue = this; let requests = []; + const processing = hotpotato.config.getStorePages; if ( hotpotato.config.getStorePages ) { requests = this.prepStorePages(hotpotato, hotpotato.config.getStorePages); hotpotato.config.getStorePages = false; @@ -14,27 +17,27 @@ export default { } else { - if (!hotpotato.config.test) this.$root.$emit("update-progress", { - text: "Fetching additional data from store pages...", - step: 0, - bar: true - }); + if ( !hotpotato.config.test ) this.$store.commit('update', [ + { key: 'progress.step', value: 0 }, + { key: 'progress.bar', value: true }, + { key: 'progress.text', value: "Fetching additional data from store pages..." }, + ]); + vue.amapxios({ requests: requests, returnCatch: true, // Returns failed steps in the step() callback in order to mark missing sote page data step: function(response, stepCallback, book, processingError) { - delete book.requestUrl; - - if (!hotpotato.config.test) vue.$root.$emit("update-progress", { text2: book.title }); - - if ( !response || response && response.status >= 400) { - book.storePageMissing = true; - } else { + + if ( _.get(book, 'requestUrl') ) delete book.requestUrl; + if (!hotpotato.config.test) vue.$store.commit('update', { key: 'progress.text2', value: _.get(book, 'title') }); + + if ( response && response.status >= 200 &&response.status < 400 ) { vue.getStorePageData(response, book, hotpotato.config.test); + } else if ( !!book ) { + book.storePageMissing = true; } - - if (!hotpotato.config.test) vue.$root.$emit("update-progress-step"); + if (!hotpotato.config.test) vue.$store.commit('update', { key: 'progress.step', add: 1 }); // if ( book.cover ) vue.$root.$emit('update-progress-thumbnail', 'https://m.media-amazon.com/images/I/'+ book.cover + '._SL120_.jpg' ); stepCallback(book); }, @@ -46,10 +49,16 @@ export default { function moveOn() { + if (!hotpotato.config.test) vue.$store.commit("resetProgress"); + if ( processing === 'wishlist' ) vue.$store.commit('update', [ + { key: 'subStep.step', value: 0 }, + { key: 'subStep.max', value: 0 }, + ]); + vue.$nextTick(function() { - if (!hotpotato.config.test) vue.$root.$emit("reset-progress"); storePagesFetched(null, hotpotato); }); + } }, @@ -150,7 +159,7 @@ export default { var tagsArray = []; var tags = audible.querySelectorAll('.bc-chip-text'); - tags.forEach( function( tag ) { + each( tags, function( tag ) { var tagObj = {}; diff --git a/src/content-script/_components/_mixins/main-step/process-wishlist.js b/src/content-script/_components/_mixins/main-step/process-wishlist.js index 11ccae1..c4e28b1 100644 --- a/src/content-script/_components/_mixins/main-step/process-wishlist.js +++ b/src/content-script/_components/_mixins/main-step/process-wishlist.js @@ -3,29 +3,47 @@ export default { getDataFromWishlist: function(hotpotato, wishlistFetched) { if ( !_.find(hotpotato.config.steps, { name: "wishlist" }) ) { - this.$root.$emit("reset-progress"); + // this.$root.$emit("reset-progress"); + this.$store.commit("resetProgress"); wishlistFetched(null, hotpotato); } else { - this.$root.$emit("update-big-step", { - title: "Wishlist", - stepAdd: 1 - }); + // this.$root.$emit("update-big-step", { + // title: "Wishlist", + // stepAdd: 1 + // }); + + // this.$root.$emit("update-sub-step", { + // step: 1, + // max: 2, + // }); - this.$root.$emit("update-progress", { - text: this.storageHasData.wishlist ? "Updating old books and adding new books..." : "Scanning wishlist for books...", - step: 0, - max: 0, - }); + // this.$root.$emit("update-progress", { + // text: this.storageHasData.wishlist ? "Updating old books and adding new books..." : "Scanning wishlist for books...", + // step: 0, + // max: 0, + // }); + + this.$store.commit('update', [ + { key: 'bigStep.title', value: 'Wishlist' }, + { key: 'bigStep.step', add: 1 }, + { key: 'subStep.step', value: 1 }, + { key: 'subStep.max', value: 2 }, + { key: 'progress.text', value: this.storageHasData.wishlist ? "Updating old books and adding new books..." : "Scanning wishlist for books..." }, + { key: 'progress.step', value: 0 }, + { key: 'progress.max', value: 0 }, + ]); const vue = this; waterfall( [ function(callback) { - vue.scrapingPrep( - vue.wishlistUrl, - function(prep) { + vue.scrapingPrep({ + url: vue.wishlistUrl, + returnResponse: true, + returnAfterFirstCall: true, + done: function(prep) { const audible = $($.parseHTML(prep.response.data)).find("div.adbl-main")[0]; const titlesLength = parseFloat( DOMPurify.sanitize(audible.querySelector("#wishlist-content-main > div > h1.bc-heading").nextElementSibling.textContent.match(/\d+/)[0]) ); delete prep.response; @@ -43,10 +61,9 @@ export default { prep.pageNumbers = _.range(1, pagesLength + 1); callback(null, prep); - }, - true, - true - ); // Stopping after the first call then doing some sleuthing around to figure out the amount of pages to scrape, rather than doing a second call (just because wisthlist loads super slow) + + } + }); // Stopping after the first call then doing some sleuthing around to figure out the amount of pages to scrape, rather than doing a second call (just because wisthlist loads super slow) }, function(prep, callback) { @@ -66,14 +83,11 @@ export default { } ], function(err, books) { - hotpotato.wishlist = books; - + + if ( books.length ) hotpotato.wishlist = books; + vue.$nextTick(function() { hotpotato.config.getStorePages = 'wishlist'; - - // if ( !vue.storageHasData.wishlist ) vue.$root.$emit("update-progress", { - // max: hotpotato.wishlist.length - // }); wishlistFetched(null, hotpotato); }); @@ -186,7 +200,8 @@ export default { if (fullScan_ALL_partialScan_NEW) { book.isNewThisRound = true; - vue.$root.$emit("update-progress-max"); + // vue.$root.$emit("update-progress-max"); + vue.$store.commit('update', { key: 'progress.max', add: 1 }); } wishlist.push(book); diff --git a/src/content-script/_components/_mixins/misc/helpers.js b/src/content-script/_components/_mixins/misc/helpers.js index 8423b59..4b37723 100644 --- a/src/content-script/_components/_mixins/misc/helpers.js +++ b/src/content-script/_components/_mixins/misc/helpers.js @@ -270,22 +270,5 @@ export default { }, - mergeBooksWithSeries: function( potatoSeries, newSeriesCollection ) { - - _.each( newSeriesCollection, function( series ) { - - const seriesExists = _.find(potatoSeries, { asin: series.asin }); - if ( seriesExists ) { - _.merge( seriesExists, series ); - } - else { - potatoSeries.push( series ); - } - - }); - - - }, - } }; diff --git a/src/content-script/_components/layout/menuScreen.vue b/src/content-script/_components/layout/menuScreen.vue index 3406a24..98eb447 100644 --- a/src/content-script/_components/layout/menuScreen.vue +++ b/src/content-script/_components/layout/menuScreen.vue @@ -391,8 +391,8 @@ export default { value: _.get( collections, 'value' ), disabled: _.get( collections, 'disabled' ), label: "Collections", - type: "is-success", - tippy: "Always a full extract, but this is a fairly quick extraction process.", + type: "is-info", + tippy: "Super quick extraction that just needs to check the first
page of each collection to find out the title and description", trash: this.hasData.collections }, { @@ -401,7 +401,7 @@ export default { disabled: _.get( wishlist, 'disabled' ), label: "Wishlist", type: "is-success", - tippy: "Books that also exist in your library are dropped
off as long as you also extract library data.", + tippy: "Similar to library extraction but series order is not fetched. Books that also exist in your library are dropped
off as long as you also extract library data.", trash: this.hasData.wishlist, // cannotAccessTippy: this.cannotAccessWishlist ? 'audible.com/login' : null, }, @@ -411,7 +411,7 @@ export default { disabled: _.get( isbn, 'disabled' ), label: "ISBN", type: "is-danger", - tippy: "You only need to extract International Standard Book Numbers (ISBN) if you want to try importing to Goodreads.
ISBNs are only fetched for books in the library.", + tippy: "International Standard Book Numbers (ISBN) are required if you want to try importing your library to Goodreads.
ISBNs are fetched for library books and not for books in the wishlist. Very slow extraction.", trash: this.hasData.isbn }, { diff --git a/src/content-script/_components/layout/scrapingProgress.vue b/src/content-script/_components/layout/scrapingProgress.vue index 92d0e58..a27d749 100644 --- a/src/content-script/_components/layout/scrapingProgress.vue +++ b/src/content-script/_components/layout/scrapingProgress.vue @@ -6,42 +6,53 @@ -
+

- {{ bigStep.title }} + {{ store.bigStep.title }} {{ bigSteps }}

-
+
-
- {{ progress.text2 }} +
+ {{ store.progress.text2 }}
- {{ progress.text }} + {{ store.progress.text }} {{ steps }} - {{ progress.textsuffix }} -
+ {{ store.progress.textsuffix }} +
- + +
+
+ {{ store.bigStep.title }} sub steps {{ subSteps }} +
+
+
- +
+ +
+ Keep this tab and the browser window on top until the extraction is completed. +
+
@@ -50,87 +61,30 @@ export default { name: "scrapingProgress", data() { return { + store: this.$store.state, imageSources: { - logo: browser.runtime.getURL( - "assets/images/audible-library-extractor-logo.svg" - ), + logo: browser.runtime.getURL("assets/images/audible-library-extractor-logo.svg"), loader: browser.runtime.getURL("assets/images/loader-64px.gif") }, - progress: { - text: null, - textSuffix: null, - text2: null, - step: 0, - max: 0, - bar: false, - thumbnail: null - }, - bigStep: { - title: "", - step: 0, - max: 0 - } }; }, computed: { steps: function() { - return this.progress.step > 0 - ? this.progress.step + " / " + this.progress.max - : this.progress.max; + return this.store.progress.step > 0 ? this.store.progress.step + " / " + this.store.progress.max : this.store.progress.max; }, bigSteps: function() { - return this.bigStep.step > 0 - ? this.bigStep.step + " / " + this.bigStep.max - : this.bigStep.max; + return this.store.bigStep.step > 0 ? this.store.bigStep.step + " / " + this.store.bigStep.max : this.store.bigStep.max; + }, + + subSteps: function() { + return this.store.subStep.step > 0 ? this.store.subStep.step + " / " + this.store.subStep.max : this.store.subStep.max; }, progressWidth: function() { - return { width: (this.progress.step / this.progress.max) * 100 + "%" }; + return { width: (this.store.progress.step / this.store.progress.max) * 100 + "%" }; } }, - - created: function() { - var vue = this; - - this.$root.$on("update-progress", function(progress) { - if (progress.text) vue.progress.text = progress.text; - if (progress.textSuffix) vue.progress.textSuffix = progress.textSuffix; - if (progress.text2) vue.progress.text2 = progress.text2; - if (progress.step > -1) vue.progress.step = progress.step; - if (progress.max > -1) vue.progress.max = progress.max; - if (progress.bar) vue.progress.bar = progress.bar; - }); - - this.$root.$on("update-progress-step", function() { - ++vue.progress.step; - }); - - this.$root.$on("update-progress-max", function() { - ++vue.progress.max; - }); - - this.$root.$on("update-progress-thumbnail", function(thumbnail) { - vue.progress.thumbnail = thumbnail; - }); - - this.$root.$on("reset-progress", function() { - vue.progress.text = null; - vue.progress.textSuffix = null; - vue.progress.text2 = null; - vue.progress.step = 0; - vue.progress.max = 0; - vue.progress.bar = false; - vue.progress.thumbnail = null; - }); - - this.$root.$on("update-big-step", function(o) { - if (o.title) vue.bigStep.title = o.title; - if (o.step > -1) vue.bigStep.step = o.step; - if (o.stepAdd) vue.bigStep.step += o.stepAdd; - if (o.max > -1) vue.bigStep.max = o.max; - }); - } }; @@ -142,19 +96,39 @@ export default { -moz-user-select: none; -ms-user-select: none; user-select: none; + text-align: center; } .ale-big-step { + display: inline-block; + text-align: center; + box-sizing: border-box; margin: 30px 0 25px; + // &.sub-step-exists { + // margin-bottom: 15px; + // } h2 { font-size: 19px; line-height: 21px; + padding: 0px 8px 8px; + box-sizing: border-box; } span { color: #999; } } +.sub-step-wrapper { + width: auto !important; + display: inline-block !important; + border-radius: 7px; + box-shadow: inset 0 3px 10px rgba(#000,.15); + padding: 5px 14px; + // margin-bottom: 15px; + margin-top: 15px; + color: #999; +} + .ale-progress { > div, > div > div { @@ -192,4 +166,13 @@ export default { font-size: 13px; color: #7a7a7a; } + +.footnote { + position: fixed; + left: 0; + right: 0; + bottom: 0; + color: #999; + z-index: 10; +} diff --git a/src/content-script/content-script-app.vue b/src/content-script/content-script-app.vue index 808c3d2..371cee6 100644 --- a/src/content-script/content-script-app.vue +++ b/src/content-script/content-script-app.vue @@ -22,12 +22,12 @@ import browser from "webextension-polyfill"; import { format as dateFormat } from "date-fns"; import DOMPurify from "dompurify"; import map from "async-es/map"; -import Url from "domurl"; +import mapLimit from "async-es/mapLimit"; import waterfall from "async-es/waterfall"; +import Url from "domurl"; global._ = _; global.$ = $; -global.asyncMap = map; global.axios = axios; global.axiosRetry = axiosRetry; global.exponentialDelay = exponentialDelay; @@ -36,16 +36,10 @@ global.browser = browser; global.dateFormat = dateFormat; global.DOMPurify = DOMPurify; global.Url = Url; +global.asyncMapLimit = mapLimit; +global.asyncMap = map; global.waterfall = waterfall; -axiosRetry(axios, { - retries: 1, - retryDelay: function(retryCount) { return 1000 * retryCount; }, - retryCondition: function(error) { - return (error && error.response && error.response.status == "429"); - } -}); - String.prototype.trimAll = function() { if (this) { return this.trim().replace(/\s+/g, " "); @@ -61,6 +55,13 @@ String.prototype.trimToColon = function() { } }; +window.each = function( array, callback ) { + if ( !array ) return null; + for (var i = 0; i < array.length; i++) { + callback( array[i], i ); + } +}; + // Components import overlay from "./_components/layout/overlay"; import menuScreen from "./_components/layout/menuScreen"; @@ -142,6 +143,197 @@ export default { // { storePageRequestUrl: "https://www.audible.com/pd/B089T5SW3N" }, // { storePageRequestUrl: "https://www.audible.com/pd/B06Y46VB4L" }, // { storePageRequestUrl: "https://www.audible.com/pd/B078X15P2P" }, + // { storePageRequestUrl: "https://www.audible.com/pd/0593104056" }, + // { storePageRequestUrl: "https://www.audible.com/pd/0785253173" }, + // { storePageRequestUrl: "https://www.audible.com/pd/0807093904" }, + // { storePageRequestUrl: "https://www.audible.com/pd/1549173499" }, + // { storePageRequestUrl: "https://www.audible.com/pd/1614967156" }, + // { storePageRequestUrl: "https://www.audible.com/pd/1645550192" }, + // { storePageRequestUrl: "https://www.audible.com/pd/1974921719" }, + // { storePageRequestUrl: "https://www.audible.com/pd/1974921778" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00CMDZUR6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D6OQ70O" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D6SYSWO" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D8DIE0E" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D8DIVQ6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D8J59NI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D8K5EFA" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D9A16QK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D9BQOZW" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00D97SJJK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DB3TXFQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DB4ZAP2" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DB6KIKC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DCCAM7Y" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DD2WFGY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DDVRFIS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DDX6EF6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DEL13V2" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DEOXKR4" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DGBT1L4" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DGCGP1W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DGCH98K" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DI952QC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DIATCXA" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DJBVBMI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DL2DAXS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DL6HROW" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00DMOB2TO" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00E825RDQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00ENL862G" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00FL2DWNO" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00FX74JKC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00GBI17ES" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00GIQZWPI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00GT2KCXI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00HJCEGKM" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00HQJUL8O" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00HU7V80W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00IP11BUY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00IXWH5KA" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00JBCTMVQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00KN2BK40" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00KN11F3W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00KTKZUWY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00LPMHE4C" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00MAKI55U" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00MEQS4F6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00OBVT4ME" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00P1RW1CC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00Q3MX9PW" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00Q5DHLBM" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00R6PULB0" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00REUDYY8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00XP35LM6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002UZL7R8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0K2D2" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0KMVY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0PXAY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0RCTY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0852C" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V087T8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V1M02W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V1ODL8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V5BBDC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V5CRYE" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V5IVHG" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V8L2UQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V8LJGS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V8MCV4" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V9Z7KG" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002VA8O5A" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B003MB94D8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B0038FYKBY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004E8IAMY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004NF9AKO" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004RCJTTA" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004SI1E00" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004TAFWT6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B0040I0LI4" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B00443X056" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B005E1GBDK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B005FM5AZI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B005IZ9C5Q" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B006YD81BC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B007SY8QKM" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B008FR70AI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B009PRFDA6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B0095Y0R8C" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01AYGLFTO" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01BFO1MLK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01BFOTLG8" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01BMLELYG" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01CT0SQBS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01D0FJOAI" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01ELX0V2S" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01ETLC4T0" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01F2BZ3OY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01II2R39W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01K4ZNTFS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01KOW9NEM" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01L9UTO24" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01MF66KQ6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01MQNGLJQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01MTRYFQH" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01N5OU58Z" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01N9Z0D30" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B011LUCFZU" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B014V81SLQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B0159LVSNC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B016QSUEJ6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B06WWLTWTS" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B06XDX2FST" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B06XWZJK47" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B06Y4F5KH3" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07CV165L5" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07D5H12L6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07D7GJ6ZL" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07DJZ7WK1" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07DSJKKLM" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07FKQFK92" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07HKS62RH" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07HKSVJQY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07LC1LF9T" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07NWWFC9S" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07P6S3QGD" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07RZMZDQH" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07SZJ35BW" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07YL79YNK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B07YSSLLXC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B071ZM4FHX" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B072JYZGWH" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B072888W6M" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B073JR7W68" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B073PJ2227" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B074SXVWNQ" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B0758LTFB4" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B077K4XXKV" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B078P67ZQK" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B08PDWTPWF" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B08S7SLKT9" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09CLKN4BV" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09D41K1JC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09F18K5XN" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09JQV7HTT" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09JYBNLCG" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09NLF95YY" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09SKGV1FC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B09SKPYNT1" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B094WGCVY7" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B016WNPN3W" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B004GI55Q6" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V0KA3O" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B002V1CDYC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B007PSX35O" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B01H45E0CC" }, + // { storePageRequestUrl: "https://www.audible.com/pd/B08ZBT2HB5" }, + ], + doSeriesTest: [ + // { series: [{asin: 'B018T5ESBE'}] }, + // { series: [{asin: 'B007D0J4H0'}] }, + // { series: [{asin: 'B00DBL75N0'}] }, + // { series: [{asin: 'B00DLM6CUG'}] }, + // { series: [{asin: 'B00DMOAB2I'}] }, + // { series: [{asin: 'B005NAD2U2'}] }, + // { series: [{asin: 'B005NAZT7Q'}] }, + // { series: [{asin: 'B006K1LRQO'}] }, + // { series: [{asin: 'B006K1O85G'}] }, + // { series: [{asin: 'B006K1OHNO'}] }, + // { series: [{asin: 'B007B4S7OE'}] }, + // { series: [{asin: 'B007SXTNLE'}] }, + // { series: [{asin: 'B007ZTL4TU'}] }, + // { series: [{asin: 'B00713SI1W'}] }, + // { series: [{asin: 'B008CQVVIO'}] }, + // { series: [{asin: 'B0083DGAGO'}] }, + // { series: [{asin: 'B015WWM7GU'}] }, + // { series: [{asin: 'B016OQVUQQ'}] }, + // { series: [{asin: 'B07HGLKLZG'}] }, + // { series: [{asin: 'B07MJTWRBW'}] }, + // { series: [{asin: 'B07S6SDQB6'}] }, + // { series: [{asin: 'B0793R8X2P'}] }, + // { series: [{asin: 'B086FRQ2L5'}] }, + // { series: [{asin: 'B09D4N723K'}] }, + // { series: [{asin: 'B006XE41AC'}] }, ], }; }, @@ -164,7 +356,7 @@ export default { // vue.init_purchaseHistoryTest(); if ( this.doStorePageTest && this.doStorePageTest.length > 0 ) vue.init_storePageTest(); - // vue.init_seriesPageTest(); + if ( this.doSeriesTest && this.doSeriesTest.length > 0 ) vue.init_seriesPageTest(); // this.scrapingPrep(this.libraryUrl, function(prep) { // console.log('PREP?', prep) @@ -205,20 +397,36 @@ export default { vue.getISBNsFromGoogleBooks, // Requires library page data ]; - vue.$root.$emit("update-big-step", { - max: config.steps ? _.filter(config.steps, function(o) { - return o.value; - }).length - : waterfallArray.length - 1 // First function is just a kind of a failsafe and doesn't count - }); + const maxSteps = config.steps ? _.filter(config.steps, function(o) { return o.value; }).length : waterfallArray.length - 1 // First function is just a kind of a failsafe and doesn't count + vue.$store.commit('update', { key: 'bigStep.max', value: maxSteps }); + + // vue.$root.$emit("update-big-step", { + // max: config.steps ? _.filter(config.steps, function(o) { + // return o.value; + // }).length + // : waterfallArray.length - 1 // First function is just a kind of a failsafe and doesn't count + // }); waterfall(waterfallArray, function(err, hotpotato) { - vue.$root.$emit("reset-progress"); - vue.$root.$emit("update-big-step", { - title: "Opening the gallery after packing up the data...", - step: 0, - max: 0 - }); + + vue.$store.commit('resetProgress'); + vue.$store.commit('update', [ + { key: 'bigStep.title', value: 'Opening the gallery after packing up the data...' }, + { key: 'bigStep.step', value: 0 }, + { key: 'bigStep.max', value: 0 }, + { key: 'subStep.step', value: 0 }, + { key: 'subStep.max', value: 0 }, + ]); + // vue.$root.$emit("reset-progress"); + // vue.$root.$emit("update-big-step", { + // title: "Opening the gallery after packing up the data...", + // step: 0, + // max: 0 + // }); + // vue.$root.$emit("update-sub-step", { + // step: 0, + // max: 0, + // }); // const configISBN = _.find(hotpotato.config.steps, { name: "isbn" }); // const foundISBNs = _.filter(hotpotato.books, 'isbns'); @@ -255,6 +463,9 @@ export default { goToOutputPage: function(hotpotato) { + // console.log('goToOutputPage', hotpotato); + // return; + let vue = this; let collections = hotpotato.collections; let archive = collections ? _.find( collections, { id: '__ARCHIVE' }) : null; @@ -389,18 +600,47 @@ export default { const vue = this; const hotpotato = { - config: { test: true, getStorePages: 'books' }, + config: { test: true, getStorePages: 'books', seriesTest: true }, books: this.doStorePageTest, }; + + // vue.getDataFromStorePages(hotpotato, function(nullBoy, result) { + // console.log( + // "%c" + "Store page test" + "", + // "background: #00bb1e; color: #fff; padding: 2px 5px; border-radius: 8px;", + // result + // ); + // }); + + const waterfallArray = [ + function(callback) { callback(null, hotpotato); }, + vue.getDataFromStorePages, // Requires library page data + vue.getDataFromSeriesPages, // Requires store page data (for fallback) + ]; + + waterfall(waterfallArray, function(err, hotpotato) { + + vue.$store.commit('resetProgress'); + vue.$store.commit('update', [ + { key: 'bigStep.title', value: 'Opening the gallery after packing up the data...' }, + { key: 'bigStep.step', value: 0 }, + { key: 'bigStep.max', value: 0 }, + ]); + // vue.$root.$emit("reset-progress"); + // vue.$root.$emit("update-big-step", { + // title: "Opening the gallery after packing up the data...", + // step: 0, + // max: 0 + // }); - vue.getDataFromStorePages(hotpotato, function(nullBoy, result) { console.log( "%c" + "Store page test" + "", "background: #00bb1e; color: #fff; padding: 2px 5px; border-radius: 8px;", - result + hotpotato ); + }); - + }, init_seriesPageTest: function() { @@ -408,10 +648,7 @@ export default { const hotpotato = { config: { seriesTest: true }, - books: [ - { series: [{asin: 'B005NB2IG0'}] }, - { series: [{asin: 'B077XNSN35'}] }, - ] + books: this.doSeriesTest, }; vue.getDataFromSeriesPages(hotpotato, function(nullBoy, result) { diff --git a/src/content-script/content-script.js b/src/content-script/content-script.js index 6c00975..ebe02fe 100644 --- a/src/content-script/content-script.js +++ b/src/content-script/content-script.js @@ -2,6 +2,7 @@ import Vue from "vue"; import App from "./content-script-app"; +import store from "./store.js"; Vue.config.productionTip = false; Vue.config.devtools = false; @@ -89,6 +90,8 @@ browser.runtime.onMessage.addListener(message => { function audibleLibraryExtractor(data) { + store.commit("fromLocalStorage"); + $('
', { id: 'audible-library-extractor'}).prependTo("body"); // Storage data is dropped immediately. I just want to know if the data exists @@ -103,6 +106,7 @@ function audibleLibraryExtractor(data) { new Vue({ el: "#audible-library-extractor", + store, render: h => { return h(App, { props: { diff --git a/src/content-script/store.js b/src/content-script/store.js new file mode 100644 index 0000000..94276bc --- /dev/null +++ b/src/content-script/store.js @@ -0,0 +1,86 @@ + +import Vue from "vue"; +import Vuex from "vuex"; +Vue.use(Vuex); + +import { + get, + map, + filter, + findIndex, + find, + each, + isArray, + assign, +} from "lodash"; + +export default new Vuex.Store({ + + state: { + progress: { + text: null, + textSuffix: null, + text2: null, + step: 0, + max: 0, + bar: false, + thumbnail: null + }, + bigStep: { + title: "", + step: 0, + max: 0 + }, + subStep: { + step: 0, + max: 0 + } + }, + + mutations: { + + fromLocalStorage: function(state) { + const lsState = JSON.parse(localStorage.getItem("ale-content-script")); + if (lsState) _.merge( state.sticky, lsState ); + }, + + update(state, config) { + + let setValues = function (config) { + config = config || {}; + if (config.key) { + let newValue = config.value; + if ( config.add ) { + const oldValue = _.get(state, config.key); + newValue = (oldValue || 0) + config.add; + } + newValue = config.freeze ? Object.freeze(newValue) : newValue; + _.set(state, config.key, newValue); + } + }; + + if (_.isArray(config)) { + _.each(config, function(conf) { + setValues(conf); + }); + } else { + setValues(config); + } + + }, + + resetProgress: function( state ) { + + state.progress.text = null; + state.progress.textSuffix = null; + state.progress.text2 = null; + state.progress.step = 0; + state.progress.max = 0; + state.progress.bar = false; + state.progress.thumbnail = null; + + }, + + } + +});