diff --git a/src/locales/scripts/bulk-download.js b/src/locales/scripts/bulk-download.js new file mode 100644 index 0000000000..1cc5dfcb8f --- /dev/null +++ b/src/locales/scripts/bulk-download.js @@ -0,0 +1,116 @@ +const { pipeline } = require("stream/promises") + +const { createWriteStream } = require("fs") + +const qs = require("qs") +const AdmZip = require("adm-zip") + +const { writeLocaleFile } = require("./utils") +const axios = require("./axios") +const jed1xJsonToJson = require("./jed1x-json-to-json") + +const LOGIN_URL = "https://login.wordpress.org/wp-login.php" +const BULK_DOWNLOAD_URL = + "https://translate.wordpress.org/exporter/meta/openverse/-do/" + +/** + * Given a username and password, login to WordPress and get the authentication + * cookies from the `Set-Cookie` header. + * + * @param log {string} - the username to log in with + * @param pwd {string} - the password for the given username + * @return {Promise} - the list of cookies in the `Set-Cookie` header + */ +const getAuthCookies = async (log, pwd) => { + const res = await axios.post( + LOGIN_URL, + qs.stringify({ + log, + pwd, + rememberme: "forever", + "wp-submit": "Log In", + redirect_to: "https://make.wordpress.org/", + }), + { + headers: { "content-type": "application/x-www-form-urlencoded" }, + maxRedirects: 0, + validateStatus: () => true, + } + ) + if ( + res.status == 302 && + res.headers["set-cookie"].join(" ").includes("wporg_logged_in") + ) { + return res.headers["set-cookie"].map((cookie) => + cookie.substring(0, cookie.indexOf(";")) + ) + } + throw new Error(`Authentication failed: server returned ${res.status}`) +} + +/** + * Fetch the ZIP of translations strings from GlotPress using the authentication + * cookies to access the page. + * + * @param cookies {string[]} - the cookies to authenticate the ZIP download + * @return {Promise}} - the path to the downloaded ZIP file + */ +const fetchBulkJed1x = async (cookies) => { + const res = await axios.get(BULK_DOWNLOAD_URL, { + headers: { cookie: cookies.join(";") }, + params: { "export-format": "jed1x" }, + responseType: "stream", + }) + const destPath = process.cwd() + "/src/locales/openverse.zip" + await pipeline(res.data, createWriteStream(destPath)) + return destPath +} + +/** + * Extract all JSON file from the given ZIP file. Their names are sanitised to + * be in the format `.json`. + * + * @param zipPath {string} - the path to the ZIP file to extract + * @return {Promise} - the outcome of writing all ZIP files + */ +const extractZip = async (zipPath) => { + const zip = new AdmZip(zipPath, undefined) + const localeJsonMap = zip + .getEntries() + .filter((entry) => entry.entryName.endsWith(".json")) + .map((entry) => { + const jed1xObj = JSON.parse(zip.readAsText(entry)) + const vueI18nObj = jed1xJsonToJson(jed1xObj) + const localeName = entry.name + .replace("meta-openverse-", "") + .replace(".jed.json", "") + return [localeName, vueI18nObj] + }) + return await Promise.all( + localeJsonMap.map((args) => writeLocaleFile(...args)) + ) +} + +/** + * Perform a bulk download of translation strings from GlotPress and extrat the + * JSON files from the ZIP archive. + * + * @return {Promise} - whether the bulk download succeeded + */ +const bulkDownload = async () => { + console.log("Performing bulk download.") + const username = process.env.GLOTPRESS_USERNAME + const password = process.env.GLOTPRESS_PASSWORD + + if (!(username && password)) { + console.log("Auth credentials not found, bulk download cancelled.") + throw new Error("Bulk download cancelled") + } + + const cookies = await getAuthCookies(username, password) + const zipPath = await fetchBulkJed1x(cookies) + const translations = await extractZip(zipPath) + console.log(`Successfully saved ${translations.length} translations.`) +} + +module.exports = bulkDownload diff --git a/src/locales/scripts/get-translations.js b/src/locales/scripts/get-translations.js index 103c541206..67c64fd95f 100644 --- a/src/locales/scripts/get-translations.js +++ b/src/locales/scripts/get-translations.js @@ -2,220 +2,16 @@ * Fetch the NGX-Translate JSON file for each supported language, * convert to our JSON format, and save in the correct folder. */ -const { writeFile } = require("fs/promises") -const { writeFileSync, createWriteStream } = require("fs") + +const { writeFileSync } = require("fs") const os = require("os") const chokidar = require("chokidar") -const qs = require("qs") -const AdmZip = require("adm-zip") - -const axios = require("./axios") -const jed1xJsonToJson = require("./jed1x-json-to-json") const { parseJson } = require("./read-i18n") -/** - * - * @typedef {"json"|"jed1x"|"ngx"} JSONFormat - * @returns - */ - -/** - * A GlotPress Output format for translation strings - * @typedef {("android"|"po"|"mo"|"resx"|"strings"|"properties"|"json"|"jed1x"|"ngx" & JSONFormat)} Format - */ - -const baseUrl = "https://translate.wordpress.org/projects/meta/openverse" -const bulkBaseUrl = "https://translate.wordpress.org/exporter/meta/openverse" -const loginUrl = "https://login.wordpress.org/wp-login.php" - -/** - * - * @param {Format} format - * @returns {(localeCode: string) => string} - */ -const makeTranslationUrl = - (format = "po") => - (localeCode = "en-gb") => - `${baseUrl}/${localeCode}/default/export-translations/?format=${format}` - -/** - * fetch a json translation from GlotPress - * @param {string} locale - */ -const fetchJed1xTranslation = (locale) => - axios - .get(makeTranslationUrl("jed1x")(locale)) - .then((res) => res.data) - .catch((err) => err.response.status) - -const replacePlaceholders = (json) => { - if (json === null) { - return null - } - if (typeof json === "string") { - return json.replace(/###([a-zA-Z-]*)###/g, "{$1}") - } - let currentJson = { ...json } - - for (const row of Object.entries(currentJson)) { - let [key, value] = row - currentJson[key] = replacePlaceholders(value) - } - return currentJson -} -/** - * Write translation strings to a file in the locale directory - * @param {string} locale - * @param {any} rawTranslations - */ -const writeLocaleFile = (locale, rawTranslations) => { - const translations = replacePlaceholders(rawTranslations) - return writeFile( - process.cwd() + `/src/locales/${locale}.json`, - JSON.stringify(translations, null, 2) + os.EOL - ) -} - -/** - * Write a file for each translation object - * @param {{[locale: string]: {[translation: string]: string}}} translationsByLocale - */ -const writeLocaleFiles = (translationsByLocale) => - Promise.all( - Object.entries(translationsByLocale).map(([locale, translations]) => - writeLocaleFile(locale, translations) - ) - ) - -// Check if an object is empty -const isEmpty = (obj) => Object.values(obj).every((x) => x === null) - -/** - * Write translation files to the "src/locales" directory from - * the supplied list of locales - * - * @param {string[]} locales - */ -const fetchAndConvertJed1xTranslations = (locales) => { - return Promise.allSettled(locales.map(fetchJed1xTranslation)) - .then((res) => { - let successfulTranslations = [] - let failedTranslations = [] - res.forEach(({ status, value }, index) => { - if (status === "fulfilled" && !isEmpty(value)) { - successfulTranslations[locales[index]] = value - } else { - failedTranslations.push(`${locales[index]} (${value})`) - } - }) - if (failedTranslations.length) { - console.log(`Failed to fetch ${failedTranslations.join(", ")}`) - } - return successfulTranslations - }) - .then((res) => { - Object.keys(res).forEach((key) => { - res[key] = jed1xJsonToJson(res[key]) - }) - return res - }) - .then(writeLocaleFiles) -} - -/** - * Get the URL to download a ZIP file containing all translation strings for all - * locales in the specified format. - * - * @param {Format} format - the - * @return {string} - */ -const makeBulkTranslationUrl = (format = "po") => - `${bulkBaseUrl}/-do/?export-format=${format}` - -/** - * Fetch the ZIP of translations strings from GlotPress using the authentication - * cookies to access the page. - * - * @param cookies {string[]} - the cookies to authenticate the ZIP download - * @return {Promise}} - the path to the downloaded ZIP file - */ -const fetchBulkJed1xTranslations = (cookies) => - axios - .get(makeBulkTranslationUrl("jed1x"), { - headers: { cookie: cookies.join("; ") }, - responseType: "stream", - }) - .then((res) => { - return new Promise((resolve, reject) => { - const dest = process.cwd() + "/src/locales/openverse.zip" - const writer = createWriteStream(dest) - res.data.pipe(writer) - writer.on("error", reject) - writer.on("close", () => { - resolve(dest) - }) - }) - }) - .catch((err) => { - console.log(err) - }) - -/** - * Given a username and password, login to WordPress and get the authentication - * cookies from the `Set-Cookie` header. - * - * @param log {string} - the username to log in with - * @param pwd {string} - the password for the given username - * @return {Promise} - the list of cookies in the `Set-Cookie` header - */ -const getAuthCookies = (log, pwd) => - axios - .post( - loginUrl, - qs.stringify({ - log, - pwd, - rememberme: "forever", - "wp-submit": "Log In", - redirect_to: "https://make.wordpress.org/", - }), - { - headers: { "content-type": "application/x-www-form-urlencoded" }, - maxRedirects: 0, - validateStatus: (status) => status === 302, // successful login results in redirect - } - ) - .then((res) => - res.headers["set-cookie"].map((cookie) => - cookie.substring(0, cookie.indexOf(";")) - ) - ) - -/** - * Extract all JSON file from the given ZIP file. Their names are sanitised to - * be in the format `.json`. - * - * @param zipPath {string} - the path to the ZIP file to extract - * @return {Promise} - the number of locales successfully downloaded - */ -const extractZip = (zipPath) => { - const zip = new AdmZip(zipPath) - return Promise.all( - zip - .getEntries() - .filter((entry) => entry.entryName.endsWith(".json")) - .map((entry) => { - const jed1x = JSON.parse(zip.readAsText(entry)) - const locale = entry.name - .replace("meta-openverse-", "") - .replace(".jed.json", "") - return [locale, jed1xJsonToJson(jed1x)] - }) - .map((args) => writeLocaleFile(...args)) - ) -} +const bulkDownload = require("./bulk-download") +const separateDownload = require("./separate-download") /** * Write `en.json` from `en.json5`. @@ -241,28 +37,16 @@ if (process.argv.includes("--watch")) { } if (!process.argv.includes("--en-only")) { - const username = process.env.GLOTPRESS_USERNAME - const password = process.env.GLOTPRESS_PASSWORD - if (username && password) { - console.log("Auth credentials found, performing bulk download.") - - getAuthCookies(username, password) - .then(fetchBulkJed1xTranslations) - .then(extractZip) - .then((res) => { - console.log(`Successfully saved ${res.length + 1} translations.`) - }) - .catch(console.error) - } else { - console.log("Auth credentials not found, performing parallel download.") - - const localeJSON = require("./wp-locales.json") - fetchAndConvertJed1xTranslations( - Object.values(localeJSON).map((i) => i.slug) - ) - .then((res) => { - console.log(`Successfully saved ${res.length + 1} translations.`) - }) - .catch(console.error) - } + bulkDownload() + .catch((err) => { + console.error(err) + return separateDownload() + }) + .catch((err) => { + console.error(err) + console.error(":'-( Downloading translations failed.") + if (process.argv.includes("--require-complete")) { + process.exitCode = 1 + } + }) } diff --git a/src/locales/scripts/separate-download.js b/src/locales/scripts/separate-download.js new file mode 100644 index 0000000000..a4ca219afa --- /dev/null +++ b/src/locales/scripts/separate-download.js @@ -0,0 +1,57 @@ +const axios = require("./axios") +const { writeLocaleFile } = require("./utils") +const jed1xJsonToJson = require("./jed1x-json-to-json") + +const DOWNLOAD_BASE_URL = + "https://translate.wordpress.org/projects/meta/openverse" + +const getTranslationUrl = (locale) => + `${DOWNLOAD_BASE_URL}/${locale}/default/export-translations/` + +const fetchJed1x = async (locale) => { + try { + const res = await axios.get(getTranslationUrl(locale), { + params: { format: "jed1x" }, + }) + return res.data + } catch (err) { + return err.response.status + } +} + +const isEmpty = (obj) => Object.values(obj).every((x) => x === null) + +const fetchJed1xAll = async (locales) => { + const results = await Promise.allSettled(locales.map(fetchJed1x)) + + let succeeded = {} + let failed = {} + results.forEach(({ status, value }, index) => { + if (status === "fulfilled" && !isEmpty(value)) { + succeeded[locales[index]] = jed1xJsonToJson(value) + } else { + failed[locales[index]] = value + } + }) + await Promise.all( + Object.entries(succeeded).map((args) => writeLocaleFile(...args)) + ) + + return [Object.keys(succeeded).length, Object.keys(failed).length] +} + +const separateDownload = async () => { + console.log("Performing parallel download.") + + const localeJson = require("./wp-locales.json") + const locales = Object.values(localeJson).map((item) => item.slug) + const [succeeded, failed] = await fetchJed1xAll(locales) + + console.log(`Successfully downloaded ${succeeded} translations.`) + if (failed) { + console.log(`Failed to download ${failed} translations.`) + throw new Error("Parallel download partially failed") + } +} + +module.exports = separateDownload diff --git a/src/locales/scripts/utils.js b/src/locales/scripts/utils.js index 2efa74c861..f4943825b2 100644 --- a/src/locales/scripts/utils.js +++ b/src/locales/scripts/utils.js @@ -1,3 +1,6 @@ +const { writeFile } = require("fs/promises") +const os = require("os") + /** * Mutates an object at the path with the value. If the path * does not exist, it is created by nesting objects along the @@ -19,3 +22,40 @@ exports.setToValue = function setValue(obj, path, value) { } o[a[0]] = value } + +/** + * Replace ###### with {}. + * + * @param json {any} - the JSON object to replace placeholders in + * @return {any} the sanitised JSON object + */ +const replacePlaceholders = (json) => { + if (json === null) { + return null + } + if (typeof json === "string") { + return json.replace(/###([a-zA-Z-]*)###/g, "{$1}") + } + let currentJson = { ...json } + + for (const row of Object.entries(currentJson)) { + let [key, value] = row + currentJson[key] = replacePlaceholders(value) + } + return currentJson +} + +exports.replacePlaceholders = replacePlaceholders + +/** + * Write translation strings to a file in the locale directory + * @param {string} locale + * @param {any} rawTranslations + */ +exports.writeLocaleFile = (locale, rawTranslations) => { + const translations = replacePlaceholders(rawTranslations) + return writeFile( + process.cwd() + `/src/locales/${locale}.json`, + JSON.stringify(translations, null, 2) + os.EOL + ) +}