feat: Add config store and preset & field types (#174)

* add config store with preset & field datatypes * fix datatype typing * Fix brittle types * using @mapeo/schema from mapeo-schema/pull/117 https://github.com/digidem/mapeo-schema/pull/117 * Only set default if field is notNull * Add tests * fix typo Co-authored-by: Andrew Chou <andrewchou@fastmail.com> * remove unnecesary return Co-authored-by: Andrew Chou <andrewchou@fastmail.com> * chore: update to @mapeo/schema@3.0.0-next.6 * fix tests * move deNullify into utils.js --------- Co-authored-by: Andrew Chou <andrewchou@fastmail.com>
digidem · Aug 15, 2023 · 9f16817 · 9f16817
1 parent 1f62404
commit 9f16817
Show file tree

Hide file tree

Showing 17 changed files with 260 additions and 120 deletions.
diff --git a/.../project/0000_sloppy_mikhail_rasputin.sql → drizzle/project/0000_special_masque.sql b/.../project/0000_sloppy_mikhail_rasputin.sql → drizzle/project/0000_special_masque.sql
@@ -12,10 +12,10 @@ CREATE TABLE `field` (
 	`tagKey` text NOT NULL,
 	`type` text NOT NULL,
 	`label` text NOT NULL,
-	`appearance` text DEFAULT 'multiline',
-	`snakeCase` integer DEFAULT false,
+	`appearance` text,
+	`snakeCase` integer,
 	`options` text,
-	`universal` integer DEFAULT false,
+	`universal` integer,
 	`placeholder` text,
 	`helperText` text,
 	`forks` text NOT NULL
@@ -58,7 +58,7 @@ CREATE TABLE `preset` (
 	`addTags` text NOT NULL,
 	`removeTags` text NOT NULL,
 	`fieldIds` text NOT NULL,
-	`icon` text,
+	`iconId` text,
 	`terms` text NOT NULL,
 	`forks` text NOT NULL
 );
diff --git a/drizzle/project/meta/0000_snapshot.json b/drizzle/project/meta/0000_snapshot.json
@@ -1,7 +1,7 @@
 {
   "version": "5",
   "dialect": "sqlite",
-  "id": "a1266e0d-e17f-41e2-a5ab-fdf6e30eaf71",
+  "id": "7252a475-6391-4179-bb17-214a01d57960",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "field_backlink": {
@@ -91,16 +91,14 @@
           "type": "text",
           "primaryKey": false,
           "notNull": false,
-          "autoincrement": false,
-          "default": "'multiline'"
+          "autoincrement": false
         },
         "snakeCase": {
           "name": "snakeCase",
           "type": "integer",
           "primaryKey": false,
           "notNull": false,
-          "autoincrement": false,
-          "default": false
+          "autoincrement": false
         },
         "options": {
           "name": "options",
@@ -114,8 +112,7 @@
           "type": "integer",
           "primaryKey": false,
           "notNull": false,
-          "autoincrement": false,
-          "default": false
+          "autoincrement": false
         },
         "placeholder": {
           "name": "placeholder",
@@ -363,8 +360,8 @@
           "notNull": true,
           "autoincrement": false
         },
-        "icon": {
-          "name": "icon",
+        "iconId": {
+          "name": "iconId",
           "type": "text",
           "primaryKey": false,
           "notNull": false,

diff --git a/drizzle/project/meta/_journal.json b/drizzle/project/meta/_journal.json
@@ -5,8 +5,8 @@
     {
       "idx": 0,
       "version": "5",
-      "when": 1691501404186,
-      "tag": "0000_sloppy_mikhail_rasputin",
+      "when": 1692098766607,
+      "tag": "0000_special_masque",
       "breakpoints": true
     }
   ]

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -82,7 +82,7 @@
     "@fastify/type-provider-typebox": "^3.3.0",
     "@hyperswarm/secret-stream": "^6.1.2",
     "@mapeo/crypto": "^1.0.0-alpha.4",
-    "@mapeo/schema": "^3.0.0-next.5",
+    "@mapeo/schema": "^3.0.0-next.6",
     "@mapeo/sqlite-indexer": "^1.0.0-alpha.5",
     "@sinclair/typebox": "^0.29.6",
     "b4a": "^1.6.3",

diff --git a/src/core-manager/index.js b/src/core-manager/index.js
@@ -12,6 +12,7 @@ import { ReplicationStateMachine } from './replication-state-machine.js'
 // are used for key derivation
 export const NAMESPACES = /** @type {const} */ ([
   'auth',
+  'config',
   'data',
   'blobIndex',
   'blob',

diff --git a/src/datastore/index.js b/src/datastore/index.js
@@ -25,6 +25,7 @@ import pDefer from 'p-defer'
 
 const NAMESPACE_SCHEMAS = /** @type {const} */ ({
   data: ['observation'],
+  config: ['preset', 'field'],
   auth: [],
 })
 
@@ -50,7 +51,7 @@ export class DataStore extends TypedEmitter {
    * @param {object} opts
    * @param {import('../core-manager/index.js').CoreManager} opts.coreManager
    * @param {TNamespace} opts.namespace
-   * @param {import('../index-writer/index.js').IndexWriter<MapeoDocTablesMap[TSchemaName]>} opts.indexWriter
+   * @param {import('../index-writer/index.js').IndexWriter} opts.indexWriter
    * @param {MultiCoreIndexer.StorageParam} opts.storage
    */
   constructor({ coreManager, namespace, indexWriter, storage }) {

diff --git a/src/datatype/index.d.ts b/src/datatype/index.d.ts
@@ -22,8 +22,8 @@ type OmitUnion<T, K extends keyof any> = T extends any ? Omit<T, K> : never
 
 export class DataType<
   TDataStore extends import('../datastore/index.js').DataStore,
-  TSchemaName extends TDataStore['schemas'][number],
-  TTable extends MapeoDocTablesMap[TSchemaName],
+  TTable extends MapeoDocTables,
+  TSchemaName extends TTable['_']['name'],
   TDoc extends MapeoDocMap[TSchemaName],
   TValue extends MapeoValueMap[TSchemaName]
 > {

diff --git a/src/datatype/index.js b/src/datatype/index.js
@@ -3,6 +3,7 @@ import { validate } from '@mapeo/schema'
 import { getTableConfig } from 'drizzle-orm/sqlite-core'
 import { eq, placeholder } from 'drizzle-orm'
 import { randomBytes } from 'node:crypto'
+import { deNullify } from '../utils.js'
 
 /**
  * @typedef {import('@mapeo/schema').MapeoDoc} MapeoDoc
@@ -187,20 +188,3 @@ export class DataType {
     return { docId, createdAt }
   }
 }
-
-/**
- * When reading from SQLite, any optional properties are set to `null`. This
- * converts `null` back to `undefined` to match the input types (e.g. the types
- * defined in @mapeo/schema)
- * @template {{}} T
- * @param {T} obj
- * @returns {import('../types.js').NullableToOptional<T>}
- */
-export function deNullify(obj) {
-  /** @type {Record<string, any>} */
-  const objNoNulls = {}
-  for (const [key, value] of Object.entries(obj)) {
-    objNoNulls[key] = value === null ? undefined : value
-  }
-  return /** @type {import('../types.js').NullableToOptional<T>} */ (objNoNulls)
-}
diff --git a/src/mapeo-project.js b/src/mapeo-project.js
@@ -6,7 +6,7 @@ import { CoreManager } from './core-manager/index.js'
 import { DataStore } from './datastore/index.js'
 import { DataType } from './datatype/index.js'
 import { IndexWriter } from './index-writer/index.js'
-import { observationTable } from './schema/project.js'
+import { fieldTable, observationTable, presetTable } from './schema/project.js'
 import RandomAccessFile from 'random-access-file'
 import RAM from 'random-access-memory'
 import Database from 'better-sqlite3'
@@ -79,10 +79,16 @@ export class MapeoProject {
       sqlite,
     })
     const indexWriter = new IndexWriter({
-      tables: [observationTable],
+      tables: [observationTable, presetTable, fieldTable],
       sqlite,
     })
     this.#dataStores = {
+      config: new DataStore({
+        coreManager: this.#coreManager,
+        namespace: 'config',
+        indexWriter,
+        storage: indexerStorage,
+      }),
       data: new DataStore({
         coreManager: this.#coreManager,
         namespace: 'data',
@@ -96,10 +102,26 @@ export class MapeoProject {
         table: observationTable,
         db,
       }),
+      preset: new DataType({
+        dataStore: this.#dataStores.config,
+        table: presetTable,
+        db,
+      }),
+      field: new DataType({
+        dataStore: this.#dataStores.config,
+        table: fieldTable,
+        db,
+      }),
     }
   }
 
   get observation() {
     return this.#dataTypes.observation
   }
+  get preset() {
+    return this.#dataTypes.preset
+  }
+  get field() {
+    return this.#dataTypes.field
+  }
 }
diff --git a/src/schema/schema-to-drizzle.js b/src/schema/schema-to-drizzle.js
@@ -80,12 +80,13 @@ export function jsonSchemaToDrizzleColumns(schema) {
         continue
       }
     }
-    const defaultValue = getDefault(value)
-    if (typeof defaultValue !== 'undefined') {
-      columns[key] = columns[key].default(defaultValue)
-    }
     if (isRequired(schema, key)) {
       columns[key] = columns[key].notNull()
+      // Only set defaults for required fields
+      const defaultValue = getDefault(value)
+      if (typeof defaultValue !== 'undefined') {
+        columns[key] = columns[key].default(defaultValue)
+      }
     }
   }
   // Not yet in @mapeo/schema

diff --git a/src/types.ts b/src/types.ts
@@ -161,3 +161,10 @@ export { Duplex }
 
 export { NoiseStream }
 export type ProtocolStream = NoiseStream & { userData: Protomux }
+
+// Unsafe type for Object.entries - you must be sure that the object does not
+// have additional properties that are not defined in the type, e.g. when using
+// a const value
+export type Entries<T> = {
+  [K in keyof T]: [K, T[K]]
+}[keyof T][]
diff --git a/src/utils.js b/src/utils.js
@@ -61,3 +61,21 @@ export async function openedNoiseSecretStream(stream) {
   await stream.opened
   return /** @type {OpenedNoiseStream | DestroyedNoiseStream} */ (stream)
 }
+
+/**
+ * When reading from SQLite, any optional properties are set to `null`. This
+ * converts `null` back to `undefined` to match the input types (e.g. the types
+ * defined in @mapeo/schema)
+ * @template {{}} T
+ * @param {T} obj
+ * @returns {import('./types.js').NullableToOptional<T>}
+ */
+
+export function deNullify(obj) {
+  /** @type {Record<string, any>} */
+  const objNoNulls = {}
+  for (const [key, value] of Object.entries(obj)) {
+    objNoNulls[key] = value === null ? undefined : value
+  }
+  return /** @type {import('./types.js').NullableToOptional<T>} */ (objNoNulls)
+}