Pre-fetch all Locize translations as app-wide static data

src/layouts/core/coreLayoutSSG.ts

@@ -8,7 +8,7 @@ import consolidateSanitizedAirtableDataset from '@/modules/core/airtable/consoli
 import fetchAirtableDataset from '@/modules/core/airtable/fetchAirtableDataset';
 import {
   getCustomer,
-  getSharedAirtableDataset,
+  getStaticAirtableDataset,
 } from '@/modules/core/airtable/getAirtableDataset';
 import prepareAndSanitizeAirtableDataset from '@/modules/core/airtable/prepareAndSanitizeAirtableDataset';
 import { AirtableSchema } from '@/modules/core/airtable/types/AirtableSchema';
@@ -17,6 +17,7 @@ import { AirtableDatasets } from '@/modules/core/data/types/AirtableDatasets';
 import { AirtableRecord } from '@/modules/core/data/types/AirtableRecord';
 import { Customer } from '@/modules/core/data/types/Customer';
 import { SanitizedAirtableDataset } from '@/modules/core/data/types/SanitizedAirtableDataset';
+import { getStaticLocizeTranslations } from '@/modules/core/i18n/getLocizeTranslations';
 import {
   DEFAULT_LOCALE,
   resolveFallbackLanguage,
@@ -62,7 +63,7 @@ const logger = createLogger({
  */
 export const getCoreStaticPaths: GetStaticPaths<CommonServerSideParams> = async (context: GetStaticPathsContext): Promise<StaticPathsOutput> => {
   const preferredLocalesOrLanguages = uniq<string>(supportedLocales.map((supportedLocale: I18nLocale) => supportedLocale.lang));
-  const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(preferredLocalesOrLanguages);
+  const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(preferredLocalesOrLanguages);
   const customer: AirtableRecord<Customer> = getCustomer(dataset);
 
   // Generate only pages for languages that have been allowed by the customer
@@ -105,7 +106,7 @@ export const getCoreStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
   const locale: string = hasLocaleFromUrl ? props?.params?.locale : DEFAULT_LOCALE; // If the locale isn't found (e.g: 404 page)
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
-  const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
+  let i18nTranslations: I18nextResources;
   let dataset: SanitizedAirtableDataset;
 
   if (preview) {
@@ -115,9 +116,11 @@ export const getCoreStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
     const datasets: AirtableDatasets = prepareAndSanitizeAirtableDataset(rawAirtableRecordsSets, airtableSchema, bestCountryCodes);
 
     dataset = consolidateSanitizedAirtableDataset(airtableSchema, datasets.sanitized);
+    i18nTranslations = await fetchTranslations(lang);
   } else {
     // When preview mode is not enabled, we fallback to the app-wide shared/static data (stale)
-    dataset = await getSharedAirtableDataset(bestCountryCodes);
+    dataset = await getStaticAirtableDataset(bestCountryCodes);
+    i18nTranslations = await getStaticLocizeTranslations(lang);
   }
 
   return {

src/layouts/core/coreLayoutSSR.ts

@@ -3,7 +3,7 @@ import { PublicHeaders } from '@/layouts/core/types/PublicHeaders';
 import { SSRPageProps } from '@/layouts/core/types/SSRPageProps';
 import {
   getCustomer,
-  getSharedAirtableDataset,
+  getStaticAirtableDataset,
 } from '@/modules/core/airtable/getAirtableDataset';
 import { Cookies } from '@/modules/core/cookiesManager/types/Cookies';
 import UniversalCookiesManager from '@/modules/core/cookiesManager/UniversalCookiesManager';
@@ -102,7 +102,7 @@ export const getCoreServerSideProps: GetServerSideProps<GetCoreServerSidePropsRe
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
   const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
-  const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(bestCountryCodes);
+  const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(bestCountryCodes);
   const customer: AirtableRecord<Customer> = getCustomer(dataset);
 
   // Do not serve pages using locales the customer doesn't have enabled

src/layouts/demo/demoLayoutSSG.ts

@@ -8,7 +8,7 @@ import consolidateSanitizedAirtableDataset from '@/modules/core/airtable/consoli
 import fetchAirtableDataset from '@/modules/core/airtable/fetchAirtableDataset';
 import {
   getCustomer,
-  getSharedAirtableDataset,
+  getStaticAirtableDataset,
 } from '@/modules/core/airtable/getAirtableDataset';
 import prepareAndSanitizeAirtableDataset from '@/modules/core/airtable/prepareAndSanitizeAirtableDataset';
 import { AirtableSchema } from '@/modules/core/airtable/types/AirtableSchema';
@@ -17,15 +17,13 @@ import { AirtableDatasets } from '@/modules/core/data/types/AirtableDatasets';
 import { AirtableRecord } from '@/modules/core/data/types/AirtableRecord';
 import { Customer } from '@/modules/core/data/types/Customer';
 import { SanitizedAirtableDataset } from '@/modules/core/data/types/SanitizedAirtableDataset';
+import { getStaticLocizeTranslations } from '@/modules/core/i18n/getLocizeTranslations';
 import {
   DEFAULT_LOCALE,
   resolveFallbackLanguage,
 } from '@/modules/core/i18n/i18n';
 import { supportedLocales } from '@/modules/core/i18n/i18nConfig';
-import {
-  fetchTranslations,
-  I18nextResources,
-} from '@/modules/core/i18n/i18nextLocize';
+import { fetchTranslations, I18nextResources } from '@/modules/core/i18n/i18nextLocize';
 import { I18nLocale } from '@/modules/core/i18n/types/I18nLocale';
 import { createLogger } from '@/modules/core/logging/logger';
 import { PreviewData } from '@/modules/core/previewMode/types/PreviewData';
@@ -62,7 +60,7 @@ const logger = createLogger({
  */
 export const getDemoStaticPaths: GetStaticPaths<CommonServerSideParams> = async (context: GetStaticPathsContext): Promise<StaticPathsOutput> => {
   const preferredLocalesOrLanguages = uniq<string>(supportedLocales.map((supportedLocale: I18nLocale) => supportedLocale.lang));
-  const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(preferredLocalesOrLanguages);
+  const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(preferredLocalesOrLanguages);
   const customer: AirtableRecord<Customer> = getCustomer(dataset);
 
   // Generate only pages for languages that have been allowed by the customer
@@ -105,7 +103,7 @@ export const getDemoStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
   const locale: string = hasLocaleFromUrl ? props?.params?.locale : DEFAULT_LOCALE; // If the locale isn't found (e.g: 404 page)
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
-  const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
+  let i18nTranslations: I18nextResources;
   let dataset: SanitizedAirtableDataset;
 
   if (preview) {
@@ -115,9 +113,11 @@ export const getDemoStaticProps: GetStaticProps<SSGPageProps, CommonServerSidePa
     const datasets: AirtableDatasets = prepareAndSanitizeAirtableDataset(rawAirtableRecordsSets, airtableSchema, bestCountryCodes);
 
     dataset = consolidateSanitizedAirtableDataset(airtableSchema, datasets.sanitized);
+    i18nTranslations = await fetchTranslations(lang);
   } else {
     // When preview mode is not enabled, we fallback to the app-wide shared/static data (stale)
-    dataset = await getSharedAirtableDataset(bestCountryCodes);
+    dataset = await getStaticAirtableDataset(bestCountryCodes);
+    i18nTranslations = await getStaticLocizeTranslations(lang);
   }
 
   return {

src/layouts/demo/demoLayoutSSR.ts

@@ -3,7 +3,7 @@ import { PublicHeaders } from '@/layouts/core/types/PublicHeaders';
 import { SSRPageProps } from '@/layouts/core/types/SSRPageProps';
 import {
   getCustomer,
-  getSharedAirtableDataset,
+  getStaticAirtableDataset,
 } from '@/modules/core/airtable/getAirtableDataset';
 import { Cookies } from '@/modules/core/cookiesManager/types/Cookies';
 import UniversalCookiesManager from '@/modules/core/cookiesManager/UniversalCookiesManager';
@@ -102,7 +102,7 @@ export const getDemoServerSideProps: GetServerSideProps<GetDemoServerSidePropsRe
   const lang: string = locale.split('-')?.[0];
   const bestCountryCodes: string[] = [lang, resolveFallbackLanguage(lang)];
   const i18nTranslations: I18nextResources = await fetchTranslations(lang); // Pre-fetches translations from Locize API
-  const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(bestCountryCodes);
+  const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(bestCountryCodes);
   const customer: AirtableRecord<Customer> = getCustomer(dataset);
 
   // Do not serve pages using locales the customer doesn't have enabled

src/modules/core/airtable/fetchRawAirtableDataset.preval.ts

@@ -26,8 +26,7 @@ import preval from 'next-plugin-preval';
  * XXX The data are therefore STALE, they're not fetched in real-time.
  *  They won't update (the app won't display up-to-date data until the next deployment, for static pages).
  *
- * @example await import('@/modules/core/airtable/fetchRawAirtableDataset.preval')
- * @example const rawAirtableRecordsSets: RawAirtableRecordsSet[] = (await import('@/modules/core/airtable/fetchRawAirtableDataset.preval')) as unknown as RawAirtableRecordsSet[];
+ * @example const rawAirtableRecordsSets: RawAirtableRecordsSet[] = await getStaticRawAirtableDataset();
  *
  * @see https://github.com/ricokahler/next-plugin-preval
  */

src/modules/core/airtable/getAirtableDataset.ts

@@ -30,9 +30,9 @@ export const getCustomer = (dataset: SanitizedAirtableDataset): AirtableRecord<C
 /**
  * Returns the whole dataset (raw), based on the app-wide static/shared/stale data fetched at build time.
  *
- * @example const rawAirtableRecordsSets: RawAirtableRecordsSet[] = await getSharedRawAirtableDataset();
+ * @example const rawAirtableRecordsSets: RawAirtableRecordsSet[] = await getStaticRawAirtableDataset();
  */
-export const getSharedRawAirtableDataset = async (): Promise<RawAirtableRecordsSet[]> => {
+export const getStaticRawAirtableDataset = async (): Promise<RawAirtableRecordsSet[]> => {
   return (await import('@/modules/core/airtable/fetchRawAirtableDataset.preval')) as unknown as RawAirtableRecordsSet[];
 };
 
@@ -42,13 +42,13 @@ export const getSharedRawAirtableDataset = async (): Promise<RawAirtableRecordsS
  * This dataset is STALE. It will not update, ever.
  * The dataset is created at build time, using the "next-plugin-preval" webpack plugin.
  *
- * @example const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(bestCountryCodes);
+ * @example const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(bestCountryCodes);
  *
  * @param preferredLocalesOrLanguages
  * @param airtableSchemaProps
  */
-export const getSharedAirtableDataset = async (preferredLocalesOrLanguages: string[], airtableSchemaProps?: GetAirtableSchemaProps): Promise<SanitizedAirtableDataset> => {
-  const rawAirtableRecordsSets: RawAirtableRecordsSet[] = await getSharedRawAirtableDataset();
+export const getStaticAirtableDataset = async (preferredLocalesOrLanguages: string[], airtableSchemaProps?: GetAirtableSchemaProps): Promise<SanitizedAirtableDataset> => {
+  const rawAirtableRecordsSets: RawAirtableRecordsSet[] = await getStaticRawAirtableDataset();
   const airtableSchema: AirtableSchema = getAirtableSchema(airtableSchemaProps);
   const datasets: AirtableDatasets = prepareAndSanitizeAirtableDataset(rawAirtableRecordsSets, airtableSchema, preferredLocalesOrLanguages);
 
@@ -87,6 +87,6 @@ export const getAirtableDataset = async (isPreviewMode: boolean, preferredLocale
     return await getLiveAirtableDataset(preferredLocalesOrLanguages, airtableSchemaProps);
   } else {
     // When preview mode is not enabled, we fallback to the app-wide shared/static data (stale)
-    return await getSharedAirtableDataset(preferredLocalesOrLanguages);
+    return await getStaticAirtableDataset(preferredLocalesOrLanguages);
   }
 };

src/modules/core/i18n/fetchLocizeTranslations.preval.ts

@@ -0,0 +1,31 @@
+import fetchLocizeTranslations from '@/modules/core/i18n/fetchLocizeTranslations';
+import preval from 'next-plugin-preval';
+
+/**
+ * Pre-fetches the Locize translations for all languages and stores the result in an cached internal JSON file.
+ * Overall, this approach allows us to have some static app-wide data that will never update, and have real-time data wherever we want.
+ *
+ * This is very useful to avoid fetching the same data for each page during the build step.
+ * By default, Next.js would call the Locize API once per page built.
+ * This was a huge pain for many reasons, because our app uses mostly static pages and we don't want those static pages to be updated.
+ *
+ * Also, even considering built time only, it was very inefficient, because Next was triggering too many API calls:
+ * - More than 40 fetch attempts (40+ demo pages)
+ * - Our in-memory cache was helping but wouldn't completely conceal the over-fetching caused by Next.js
+ * - Locize API has on-demand pricing, each call costs us money
+ *
+ * The shared/static dataset is accessible to:
+ * - All components
+ * - All pages (both getStaticProps and getStaticPaths, and even in getServerSideProps is you wish to!)
+ * - All API endpoints
+ *
+ * XXX The data are therefore STALE, they're not fetched in real-time.
+ *  They won't update (the app won't display up-to-date data until the next deployment, for static pages).
+ *
+ * @example const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+ *
+ * @see https://github.com/ricokahler/next-plugin-preval
+ */
+export const locizeTranslations = preval(fetchLocizeTranslations());
+
+export default locizeTranslations;

src/modules/core/i18n/fetchLocizeTranslations.ts

@@ -0,0 +1,42 @@
+import { supportedLocales } from '@/modules/core/i18n/i18nConfig';
+import {
+  fetchTranslations,
+  I18nextResources,
+} from '@/modules/core/i18n/i18nextLocize';
+import { I18nLocale } from '@/modules/core/i18n/types/I18nLocale';
+import { createLogger } from '@/modules/core/logging/logger';
+
+const fileLabel = 'modules/core/i18n/fetchLocizeTranslations';
+const logger = createLogger({
+  fileLabel,
+});
+
+export type LocizeTranslationByLang = {
+  [lang: string]: I18nextResources;
+}
+
+/**
+ * Fetches the Locize API.
+ * Invoked by fetchLocizeTranslations.preval.preval.ts file at build time (during Webpack bundling).
+ *
+ * XXX Must be a single export file otherwise it can cause issues - See https://github.com/ricokahler/next-plugin-preval/issues/19#issuecomment-848799473
+ *
+ * XXX We opinionately decided to use the "lang" (e.g: 'en') as Locize index, but it could also be the "name" (e.g: 'en-US'), it depends on your business requirements!
+ *  (lang is simpler)
+ */
+export const fetchLocizeTranslations = async (): Promise<LocizeTranslationByLang> => {
+  const translationsByLocale: LocizeTranslationByLang = {};
+  const promises: Promise<any>[] = [];
+
+  supportedLocales.map((supportedLocale: I18nLocale) => {
+    promises.push(fetchTranslations(supportedLocale?.lang));
+  });
+
+  // Run all promises in parallel and compute results into the dataset
+  const results: I18nextResources[] = await Promise.all(promises);
+  results.map((i18nextResources: I18nextResources, index) => translationsByLocale[supportedLocales[index]?.lang] = i18nextResources);
+
+  return translationsByLocale;
+};
+
+export default fetchLocizeTranslations;

src/modules/core/i18n/getLocizeTranslations.ts

@@ -0,0 +1,30 @@
+import { LocizeTranslationByLang } from '@/modules/core/i18n/fetchLocizeTranslations';
+import { I18nextResources } from '@/modules/core/i18n/i18nextLocize';
+import { createLogger } from '@/modules/core/logging/logger';
+
+const fileLabel = 'modules/core/i18n/getLocizeTranslations';
+const logger = createLogger({
+  fileLabel,
+});
+
+/**
+ * Returns all translations (indexed by language), based on the app-wide static/shared/stale data fetched at build time.
+ *
+ * @example const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+ */
+export const getAllStaticLocizeTranslations = async (): Promise<LocizeTranslationByLang> => {
+  return (await import('@/modules/core/i18n/fetchLocizeTranslations.preval')) as unknown as LocizeTranslationByLang;
+};
+
+/**
+ * Returns all translations for one language, based on the app-wide static/shared/stale data fetched at build time.
+ *
+ * @example const i18nTranslations: I18nextResources = await getStaticLocizeTranslations(lang);
+ *
+ * @param lang
+ */
+export const getStaticLocizeTranslations = async (lang: string): Promise<I18nextResources> => {
+  const allStaticLocizeTranslations = await getAllStaticLocizeTranslations();
+
+  return allStaticLocizeTranslations?.[lang];
+};

src/modules/core/i18n/i18nextLocize.ts

@@ -253,7 +253,7 @@ export const fetchBaseTranslations = async (lang: string): Promise<I18nextResour
 };
 
 /**
- * Fetches the translations that are specific to the customer (its own translations variation)
+ * Fetches the translations that are specific to the customer (their own translations variation)
  *
  * @param lang
  */

src/modules/core/i18n/middlewares/localeMiddleware.ts

@@ -1,6 +1,6 @@
 import {
   getCustomer,
-  getSharedAirtableDataset,
+  getStaticAirtableDataset,
 } from '@/modules/core/airtable/getAirtableDataset';
 import { AirtableRecord } from '@/modules/core/data/types/AirtableRecord';
 import { Customer } from '@/modules/core/data/types/Customer';
@@ -41,7 +41,7 @@ export const localeMiddleware = async (req: NextApiRequest, res: NextApiResponse
   const detections: string[] = acceptLanguageHeaderLookup(req) || [];
   let localeFound; // Will contain the most preferred browser locale (e.g: fr-FR, fr, en-US, en, etc.)
   const preferredLocalesOrLanguages = uniq<string>(supportedLocales.map((supportedLocale: I18nLocale) => supportedLocale.lang));
-  const dataset: SanitizedAirtableDataset = await getSharedAirtableDataset(preferredLocalesOrLanguages);
+  const dataset: SanitizedAirtableDataset = await getStaticAirtableDataset(preferredLocalesOrLanguages);
   const customer: AirtableRecord<Customer> = getCustomer(dataset);
 
   if (detections && !!size(detections)) {