feat: Add lang option to DataType read operations (#473)

* feat: Add `lang` option to DataType read operations * update typescript defs * add getMany support * initial implementation of `TranslationApi` * * add @types/json-stable-strinfify * start writing unit tests * error handling for missing translation * fix error in sql query * * remove `message` from object when hashing * fix logic error when `put` * handle unexisting key in the indexing (by creating a new Set) * make `fieldRef` and `regionCode` optional when `get` * improve `get()` by ignoring not translated languages, improve typing * * make `index` private, run it on `put` * expose cache map through symbol and getter * improve `get` signature (by making `fieldRef` and `regionCode` optional) * add more unit tests * revert `index` as private method, update tests * update magic-bytes manually * add e2e/translation-api.js and expose translation api in mapeo project * * add translationTable to index writer * add translationDoc to entries for batching indexer * move decoding of translationDoc into `.index` function in translationApi * rever changes to `.index` method (better to accept doc than block) * first e2e tests * revert regionCode fallback (since its handled in an upper layer) * use default config for translationApi e2e tests, test with a bunch of translations * add translation fixtures * add check of expected translation * improve test messages * add assertion of matching preset docId with translation docIdRef * add tests and fixture for fields * Integrate actual translation-api in DataType * add type to translation * Promise.all translation on getMany * datatype clean, type translationApi in tests * add fallback when not matching `regionCode`, test that * re-add misterious dissapearing //ts-expect error on datatype * Apply suggestions from code review * Have better typing on `getByDocId` result * Avoid extra translation lookup * remove ts-ignore Co-authored-by: Evan Hahn <me@evanhahn.com> * revert removal of translation object clone in datatype * check if `fieldRef` is string before setting the property * Fix "use before define" issue with DataType & TranslationApi (#604) `DataType` needs access to `TranslationApi.prototype.get`, but `TranslationApi` needs access to `DataType`. To avoid this circular reference, pass a `getTranslations` function into `DataType`. * add e2e tests * add e2e tests for presets * chore: fix type errors in translation api e2e test (#619) See [this comment](https://github.com/digidem/mapeo-core-next/pull/473/files/a51f709ca5edf7eec82c37ab922eab81433119c3#diff-8d53b37e24956c77fbf7b47691ded78c4b5c35010b94ad0b7e3d4dc49bcb6419). --------- Co-authored-by: Tomás Ciccola <tciccola@digital-democracy.com> Co-authored-by: tomasciccola <117094913+tomasciccola@users.noreply.github.com> Co-authored-by: Evan Hahn <me@evanhahn.com>
digidem · May 7, 2024 · 7f8fcdd · 7f8fcdd
1 parent a84e89a
commit 7f8fcdd
Show file tree

Hide file tree

Showing 9 changed files with 402 additions and 19 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -149,12 +149,14 @@
     "@mapeo/sqlite-indexer": "1.0.0-alpha.8",
     "@sinclair/typebox": "^0.29.6",
     "b4a": "^1.6.3",
+    "bcp-47": "^2.1.0",
     "better-sqlite3": "^8.7.0",
     "big-sparse-array": "^1.0.3",
     "bogon": "^1.1.0",
     "compact-encoding": "^2.12.0",
     "corestore": "^6.8.4",
     "debug": "^4.3.4",
+    "dot-prop": "^8.0.2",
     "drizzle-orm": "^0.30.8",
     "fastify": ">= 4",
     "fastify-plugin": "^4.5.1",

diff --git a/src/datatype/index.d.ts b/src/datatype/index.d.ts
@@ -13,6 +13,7 @@ import { SQLiteSelectBase } from 'drizzle-orm/sqlite-core'
 import { RunResult } from 'better-sqlite3'
 import type Hypercore from 'hypercore'
 import { TypedEmitter } from 'tiny-typed-emitter'
+import TranslationApi from '../translation-api.js'
 
 type MapeoDocTableName = `${MapeoDoc['schemaName']}Table`
 type GetMapeoDocTables<T> = T[keyof T & MapeoDocTableName]
@@ -52,11 +53,13 @@ export class DataType<
     table,
     getPermissions,
     db,
+    getTranslations,
   }: {
     table: TTable
     dataStore: TDataStore
     db: import('drizzle-orm/better-sqlite3').BetterSQLite3Database
     getPermissions?: () => any
+    getTranslations: TranslationApi['get']
   })
 
   get [kTable](): TTable
@@ -79,12 +82,16 @@ export class DataType<
     >
   >(value: T): Promise<TDoc & { forks: string[] }>
 
-  getByDocId(docId: string): Promise<TDoc & { forks: string[] }>
+  getByDocId(
+    docId: string,
+    opts?: { lang?: string }
+  ): Promise<TDoc & { forks: string[] }>
 
-  getByVersionId(versionId: string): Promise<TDoc>
+  getByVersionId(versionId: string, opts?: { lang?: string }): Promise<TDoc>
 
   getMany(opts?: {
     includeDeleted?: boolean
+    lang?: string
   }): Promise<Array<TDoc & { forks: string[] }>>
 
   update<

diff --git a/src/datatype/index.js b/src/datatype/index.js
@@ -7,6 +7,8 @@ import { noop, deNullify } from '../utils.js'
 import { NotFoundError } from '../errors.js'
 import crypto from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
+import { parse as parseBCP47 } from 'bcp-47'
+import { setProperty, getProperty } from 'dot-prop'
 
 /**
  * @typedef {import('@mapeo/schema').MapeoDoc} MapeoDoc
@@ -70,22 +72,25 @@ export class DataType extends TypedEmitter {
   #schemaName
   #sql
   #db
+  #getTranslations
 
   /**
    *
    * @param {object} opts
    * @param {TTable} opts.table
    * @param {TDataStore} opts.dataStore
    * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.db
+   * @param {import('../translation-api.js').default['get']} opts.getTranslations
    * @param {() => any} [opts.getPermissions]
    */
-  constructor({ dataStore, table, getPermissions, db }) {
+  constructor({ dataStore, table, getPermissions, db, getTranslations }) {
     super()
     this.#dataStore = dataStore
     this.#table = table
     this.#schemaName = /** @type {TSchemaName} */ (getTableConfig(table).name)
     this.#getPermissions = getPermissions
     this.#db = db
+    this.#getTranslations = getTranslations
     this.#sql = {
       getByDocId: db
         .select()
@@ -172,26 +177,73 @@ export class DataType extends TypedEmitter {
 
   /**
    * @param {string} docId
+   * @param {{ lang?: string }} [opts]
    */
-  async getByDocId(docId) {
+  async getByDocId(docId, { lang } = {}) {
     await this.#dataStore.indexer.idle()
-    const result = this.#sql.getByDocId.get({ docId })
+    const result = /** @type {undefined | MapeoDoc} */ (
+      this.#sql.getByDocId.get({ docId })
+    )
     if (!result) throw new NotFoundError()
-    return deNullify(result)
+    return this.#translate(deNullify(result), { lang })
   }
 
-  /** @param {string} versionId */
-  async getByVersionId(versionId) {
-    return this.#dataStore.read(versionId)
+  /**
+   * @param {string} versionId
+   * @param {{ lang?: string }} [opts]
+   */
+  async getByVersionId(versionId, { lang } = {}) {
+    const result = await this.#dataStore.read(versionId)
+    return this.#translate(result, { lang })
   }
 
-  /** @param {{ includeDeleted?: boolean }} [opts] */
-  async getMany({ includeDeleted = false } = {}) {
+  /**
+   * @param {MapeoDoc} doc
+   * @param {{ lang?: string }} [opts]
+   */
+  async #translate(doc, { lang } = {}) {
+    if (!lang) return doc
+
+    const { language, region } = parseBCP47(lang)
+    if (!language) return doc
+    const translatedDoc = JSON.parse(JSON.stringify(doc))
+
+    let value = {
+      languageCode: language,
+      schemaNameRef: translatedDoc.schemaName,
+      docIdRef: translatedDoc.docId,
+      regionCode: region !== null ? region : undefined,
+    }
+    let translations = await this.#getTranslations(value)
+    // if passing a region code returns no matches,
+    // fallback to matching only languageCode
+    if (translations.length === 0 && value.regionCode) {
+      value.regionCode = undefined
+      translations = await this.#getTranslations(value)
+    }
+
+    for (let translation of translations) {
+      if (typeof getProperty(doc, translation.fieldRef) === 'string') {
+        setProperty(doc, translation.fieldRef, translation.message)
+      }
+    }
+    return doc
+  }
+
+  /** @param {{ includeDeleted?: boolean, lang?: string }} [opts] */
+  async getMany({ includeDeleted = false, lang } = {}) {
     await this.#dataStore.indexer.idle()
     const rows = includeDeleted
       ? this.#sql.getManyWithDeleted.all()
       : this.#sql.getMany.all()
-    return rows.map((doc) => deNullify(doc))
+    return await Promise.all(
+      rows.map(
+        async (doc) =>
+          await this.#translate(deNullify(/** @type {MapeoDoc} */ (doc)), {
+            lang,
+          })
+      )
+    )
   }
 
   /**

diff --git a/src/mapeo-project.js b/src/mapeo-project.js
@@ -83,6 +83,7 @@ export class MapeoProject extends TypedEmitter {
   #memberApi
   #iconApi
   #syncApi
+  /** @type {TranslationApi} */
   #translationApi
   #l
 
@@ -177,6 +178,7 @@ export class MapeoProject extends TypedEmitter {
       },
       logger: this.#l,
     })
+
     this.#dataStores = {
       auth: new DataStore({
         coreManager: this.#coreManager,
@@ -201,56 +203,71 @@ export class MapeoProject extends TypedEmitter {
         storage: indexerStorage,
       }),
     }
+
+    /** @type {typeof TranslationApi.prototype.get} */
+    const getTranslations = (...args) => this.$translation.get(...args)
     this.#dataTypes = {
       observation: new DataType({
         dataStore: this.#dataStores.data,
         table: observationTable,
         db,
+        getTranslations,
       }),
       track: new DataType({
         dataStore: this.#dataStores.data,
         table: trackTable,
         db,
+        getTranslations,
       }),
       preset: new DataType({
         dataStore: this.#dataStores.config,
         table: presetTable,
         db,
+        getTranslations,
       }),
       field: new DataType({
         dataStore: this.#dataStores.config,
         table: fieldTable,
         db,
+        getTranslations,
       }),
       projectSettings: new DataType({
         dataStore: this.#dataStores.config,
         table: projectSettingsTable,
         db: sharedDb,
+        getTranslations,
       }),
       coreOwnership: new DataType({
         dataStore: this.#dataStores.auth,
         table: coreOwnershipTable,
         db,
+        getTranslations,
       }),
       role: new DataType({
         dataStore: this.#dataStores.auth,
         table: roleTable,
         db,
+        getTranslations,
       }),
       deviceInfo: new DataType({
         dataStore: this.#dataStores.config,
         table: deviceInfoTable,
         db,
+        getTranslations,
       }),
       icon: new DataType({
         dataStore: this.#dataStores.config,
         table: iconTable,
         db,
+        getTranslations,
       }),
       translation: new DataType({
         dataStore: this.#dataStores.config,
         table: translationTable,
         db,
+        getTranslations: () => {
+          throw new Error('Cannot get translation for translations')
+        },
       }),
     }
     const identityKeypair = keyManager.getIdentityKeypair()