Skip to content

Commit

Permalink
refactor(build): make Document.findAll() asynchronous (#8302)
Browse files Browse the repository at this point in the history 
This may help to improve performance.
  • Loading branch information
@queengooborg
queengooborg authored Mar 6, 2023
1 parent 084777e commit d2abaad
Show file tree
Hide file tree
Showing 8 changed files with 48 additions and 42 deletions.
4 changes: 2 additions & 2 deletions build/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ async function buildDocumentInteractive(
}

if (!interactive) {
const translations = translationsOf(document.metadata);
const translations = await translationsOf(document.metadata);
if (translations && translations.length > 0) {
document.translations = translations;
} else {
Expand Down Expand Up @@ -133,7 +133,7 @@ async function buildDocuments(

const metadata: GlobalMetadata = {};

const documents = Document.findAll(findAllOptions);
const documents = await Document.findAll(findAllOptions);
const progressBar = new cliProgress.SingleBar(
{},
cliProgress.Presets.shades_grey
Expand Down
24 changes: 13 additions & 11 deletions content/document.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,35 @@ import path from "node:path";
import * as Document from "./document.js";

describe("Document.findAll()", () => {
it("should always return files that exist", () => {
const filePaths = [...Document.findAll().iterPaths()];
it("should always return files that exist", async () => {
const filePaths = [...(await Document.findAll()).iterPaths()];
expect(filePaths.every((value) => fs.existsSync(value))).toBeTruthy();
});

it("all files should be either index.html or index.md", () => {
const filePaths = [...Document.findAll().iterPaths()];
it("all files should be either index.html or index.md", async () => {
const filePaths = [...(await Document.findAll()).iterPaths()];
expect(
filePaths.every(
(value) => value.endsWith("index.html") || value.endsWith("index.md")
)
).toBeTruthy();
});

it("searching by specific file", () => {
const filePaths = [...Document.findAll().iterPaths()];
it("searching by specific file", async () => {
const filePaths = [...(await Document.findAll()).iterPaths()];
const randomFile = filePaths[Math.floor(Math.random() * filePaths.length)];
const specificFilePaths = [
...Document.findAll({ files: new Set([randomFile]) }).iterPaths(),
...(await Document.findAll({ files: new Set([randomFile]) })).iterPaths(),
];
expect(specificFilePaths).toHaveLength(1);
expect(specificFilePaths[0]).toBe(randomFile);
});

it("searching by specific locales", () => {
it("searching by specific locales", async () => {
const specificFilePaths = [
...Document.findAll({ locales: new Map([["fr", true]]) }).iterPaths(),
...(
await Document.findAll({ locales: new Map([["fr", true]]) })
).iterPaths(),
];
expect(
specificFilePaths.every((filePath) =>
Expand All @@ -39,9 +41,9 @@ describe("Document.findAll()", () => {
).toBeTruthy();
});

it("searching by specific folders", () => {
it("searching by specific folders", async () => {
const specificFilePaths = [
...Document.findAll({ folderSearch: "/foo/" }).iterPaths(),
...(await Document.findAll({ folderSearch: "/foo/" })).iterPaths(),
];
expect(
specificFilePaths.every((filePath) =>
Expand Down
5 changes: 3 additions & 2 deletions content/document.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,7 @@ export function findByURL(
return doc;
}

export function findAll({
export async function findAll({
files = new Set<string>(),
folderSearch = null,
locales = new Map(),
Expand Down Expand Up @@ -514,7 +514,8 @@ export function findAll({
return true;
})
.crawl(root);
filePaths.push(...(api.sync() as PathsOutput));
const output: PathsOutput = await api.withPromise();
filePaths.push(...output);
}
return {
count: filePaths.length,
Expand Down
8 changes: 4 additions & 4 deletions content/translations.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ const LANGUAGES = new Map(

const TRANSLATIONS_OF = new Map();

function gatherTranslations() {
const iter = Document.findAll().iterDocs();
async function gatherTranslations() {
const iter = (await Document.findAll()).iterDocs();
for (const {
metadata: { slug, locale, title },
} of iter) {
Expand All @@ -37,11 +37,11 @@ function gatherTranslations() {
}
}

export function translationsOf({ slug, locale: currentLocale }) {
export async function translationsOf({ slug, locale: currentLocale }) {
if (TRANSLATIONS_OF.size === 0) {
const label = "Time to gather all translations";
console.time(label);
gatherTranslations();
await gatherTranslations();
console.timeEnd(label);
}
const translations = TRANSLATIONS_OF.get(slug.toLowerCase());
Expand Down
2 changes: 1 addition & 1 deletion markdown/m2h/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ program
.argument("[folder]", "convert by folder")
.action(
tryOrExit(async ({ args, options }) => {
const all = Document.findAll({
const all = await Document.findAll({
folderSearch: args.folder,
locales: buildLocaleMap(options.locale),
});
Expand Down
33 changes: 18 additions & 15 deletions server/translations.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ function packageTranslationDifferences(translationDifferences) {
}

const _foundDocumentsCache = new Map();
export function findDocuments({ locale }) {
export async function findDocuments({ locale }) {
const counts = {
// Number of documents found that aren't skipped
found: 0,
Expand All @@ -76,7 +76,7 @@ export function findDocuments({ locale }) {
const documents = [];

const t1 = new Date();
const documentsFound = Document.findAll({
const documentsFound = await Document.findAll({
locales: new Map([[locale, true]]),
});
counts.total = documentsFound.count;
Expand Down Expand Up @@ -196,7 +196,7 @@ function getDocument(filePath) {

const _defaultLocaleDocumentsCache = new Map();

function gatherL10NstatsSection({
async function gatherL10NstatsSection({
locale,
mdnSection = "/",
subSections = [],
Expand Down Expand Up @@ -272,7 +272,7 @@ function gatherL10NstatsSection({
locale + mdnSection.toLowerCase() + (mdnSection.endsWith("/") ? "" : "/")
);

const foundTranslations = Document.findAll({
const foundTranslations = await Document.findAll({
locales: new Map([[locale, true]]),
folderSearch,
});
Expand All @@ -295,7 +295,7 @@ function gatherL10NstatsSection({
(mdnSection.endsWith("/") ? "" : "/")
);

const foundDefaultLocale = Document.findAll({
const foundDefaultLocale = await Document.findAll({
locales: new Map([[DEFAULT_LOCALE.toLowerCase(), true]]),
folderSearch: folderSearchDefaultLocale,
});
Expand Down Expand Up @@ -404,7 +404,7 @@ function gatherL10NstatsSection({

const _detailsSectionCache = new Map();

function buildL10nDashboard({
async function buildL10nDashboard({
locale,
section,
}: {
Expand All @@ -420,10 +420,13 @@ function buildL10nDashboard({
}
const sectionDirPath = replaceSepPerOS(section);
const defaultLocaleDocs = [
...Document.findAll({
locales: new Map([[DEFAULT_LOCALE.toLowerCase(), true]]),
folderSearch: DEFAULT_LOCALE.toLowerCase() + sectionDirPath.toLowerCase(),
}).iterDocs(),
...(
await Document.findAll({
locales: new Map([[DEFAULT_LOCALE.toLowerCase(), true]]),
folderSearch:
DEFAULT_LOCALE.toLowerCase() + sectionDirPath.toLowerCase(),
})
).iterDocs(),
];

const subSectionsStartingWith = defaultLocaleDocs
Expand All @@ -449,7 +452,7 @@ function buildL10nDashboard({
})
.map((s) => "/" + s);

const l10nStatsSection = gatherL10NstatsSection({
const l10nStatsSection = await gatherL10NstatsSection({
locale,
mdnSection: section,
subSections,
Expand Down Expand Up @@ -539,7 +542,7 @@ router.get("/", async (req, res) => {
return res.status(500).send("CONTENT_TRANSLATED_ROOT not set");
}

const countsByLocale = countFilesByLocale();
const countsByLocale = await countFilesByLocale();

const locales = [...VALID_LOCALES]
.map(([localeLC, locale]) => {
Expand All @@ -557,7 +560,7 @@ router.get("/", async (req, res) => {
res.json({ locales });
});

function countFilesByLocale() {
async function countFilesByLocale() {
const counts = new Map();
let strip = CONTENT_TRANSLATED_ROOT;
if (!strip.endsWith(path.sep)) {
Expand Down Expand Up @@ -596,7 +599,7 @@ router.get("/differences", async (req, res) => {

const label = `Find all translated documents (${locale})`;
console.time(label);
const found = findDocuments({ locale });
const found = await findDocuments({ locale });
console.timeEnd(label);
res.json(found);
});
Expand Down Expand Up @@ -641,7 +644,7 @@ router.get("/dashboard", async (req, res) => {

const label = `Gather stat for ${locale} and section ${section}`;
console.time(label);
const data = buildL10nDashboard({ locale, section });
const data = await buildL10nDashboard({ locale, section });
console.timeEnd(label);
res.json(data);
});
8 changes: 4 additions & 4 deletions testing/tests/destructive.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,10 @@ describe("fixing flaws", () => {
.split("\n")
.filter((line) => regexPattern.test(line));
expect(dryRunNotices).toHaveLength(4);
expect(dryRunNotices[0]).toContain(path.join(pattern, "bad_pre_tags"));
expect(dryRunNotices[1]).toContain(path.join(pattern, "deprecated_macros"));
expect(dryRunNotices[2]).toContain(path.join(pattern, "images"));
expect(dryRunNotices[3]).toContain(pattern);
expect(dryRunNotices[0]).toContain(pattern);
expect(dryRunNotices[1]).toContain(path.join(pattern, "bad_pre_tags"));
expect(dryRunNotices[2]).toContain(path.join(pattern, "deprecated_macros"));
expect(dryRunNotices[3]).toContain(path.join(pattern, "images"));
const dryrunFiles = getChangedFiles(tempContentDir);
expect(dryrunFiles).toHaveLength(0);
});
Expand Down
6 changes: 3 additions & 3 deletions tool/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,7 @@ program
tryOrExit(async ({ args, options }: FixFlawsActionParameters) => {
const { fixFlawsTypes } = args;
const { locale, fileTypes } = options;
const allDocs = Document.findAll({
const allDocs = await Document.findAll({
locales: new Map([[locale.toLowerCase(), true]]),
});
for (const document of allDocs.iterDocs()) {
Expand Down Expand Up @@ -771,7 +771,7 @@ program
if (!fs.existsSync(CONTENT_TRANSLATED_ROOT)) {
throw new Error(`${CONTENT_TRANSLATED_ROOT} does not exist`);
}
const documents = Document.findAll();
const documents = await Document.findAll();
if (!documents.count) {
throw new Error("No documents to analyze");
}
Expand Down Expand Up @@ -1001,7 +1001,7 @@ if (Mozilla && !Mozilla.dntEnabled()) {
.map((m) => `"${m}"`)
.join(", ")} within content folder(s) matching "${foldersearch}"`
);
const documents = Document.findAll({
const documents = await Document.findAll({
folderSearch: foldersearch,
});
if (!documents.count) {
Expand Down

0 comments on commit d2abaad

Please sign in to comment.