feat: type-safe encode function [3/6] (#74)

* WIP: type-safe encode function * WIP: encode function skeleton * feat: start working on lib/encode-conversions.ts, on observation * Add tag conversion for decode Co-authored-by: tomasciccola <tomasciccola@users.noreply.github.com> * feat: convertField and convertProject on encode-converstions.ts I realize Project has a missing acording to the revition we did called `defaultPresets`. I'll try to add it and see how it goes... * feat: add (commented for now) convertPreset to encode-convertions * fix up encode convert functions * Add preset encode/decode --------- Co-authored-by: Tomás Ciccola <tciccola@digital-democracy.com> Co-authored-by: tomasciccola <tomasciccola@users.noreply.github.com>
digidem · Aug 1, 2023 · 11291ce · 11291ce
1 parent 096a669
commit 11291ce
Show file tree

Hide file tree

Showing 15 changed files with 477 additions and 72 deletions.
diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -13,5 +13,5 @@ plugins:
       # - useOptionals=none
       - stringEnums=true
       - enumsAsLiterals=true
-      - outputIndex=true
+      # - outputIndex=true
       - oneof=unions
diff --git a/proto/project/v1.proto b/proto/project/v1.proto
@@ -13,4 +13,18 @@ message Project_1 {
   Common_1 common = 1;
 
   string name = 5;
+  /*
+  // You can't use an enum as a key in a map in protobufs :(
+  enum DefaultPresetsKey {
+    point = 0;
+    vertex = 1;
+    line = 2;
+    area = 3;
+    relation = 4;
+  };
+  message DefaultPresetsValue {
+    repeated string defaultPresetsValue = 1;
+  };
+  map<DefaultPresetsKey,DefaultPresetsValue> defaultPresetsValue= 6;
+  */
 }
diff --git a/schema/observation/v5.json b/schema/observation/v5.json
@@ -7,6 +7,7 @@
     "position": {
       "description": "Position details",
       "type": "object",
+      "required": ["mocked"],
       "properties": {
         "timestamp": {
           "description": "Timestamp of when the current position was obtained",
@@ -54,13 +55,13 @@
     },
     "lat": {
       "description": "latitude of the observation",
-      "type": ["number", "null"],
+      "type": "number",
       "minimum": -90,
       "maximum": 90
     },
     "lon": {
       "description": "longitude of the observation",
-      "type": ["number", "null"],
+      "type": "number",
       "minimum": -180,
       "maximum": 180
     },
@@ -187,6 +188,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["schemaName", "tags"],
+  "required": ["schemaName", "tags", "refs", "attachments"],
   "additionalProperties": false
 }
diff --git a/schema/preset/v2.json b/schema/preset/v2.json
@@ -3,6 +3,47 @@
   "$id": "http://mapeo.world/schemas/preset/v2.json",
   "title": "Preset",
   "description": "Presets define how map entities are displayed to the user. They define the icon used on the map, and the fields / questions shown to the user when they create or edit the entity on the map. The `tags` property of a preset is used to match the preset with observations, nodes, ways and relations. If multiple presets match, the one that matches the most tags is used.",
+  "definitions": {
+    "tags": {
+      "type": "object",
+      "properties": {},
+      "additionalProperties": {
+        "anyOf": [
+          {
+            "type": "boolean"
+          },
+          {
+            "type": "number"
+          },
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          },
+          {
+            "type": "array",
+            "items": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  },
   "type": "object",
   "properties": {
     "schemaName": {
@@ -17,7 +58,6 @@
     "geometry": {
       "description": "Valid geometry types for the feature - this preset will only match features of this geometry type `\"point\", \"vertex\", \"line\", \"area\", \"relation\"`",
       "type": "array",
-      "minItems": 1,
       "uniqueItems": true,
       "items": {
         "type": "string",
@@ -26,23 +66,17 @@
     },
     "tags": {
       "description": "The tags are used to match the preset to existing map entities. You can match based on multiple tags E.g. if you have existing points with the tags `nature:tree` and `species:oak` then you can add both these tags here in order to match only oak trees.",
-      "type": "object",
-      "properties": {},
-      "additionalProperties": true
+      "$ref": "#/definitions/tags"
     },
     "addTags": {
       "description": "Tags that are added when changing to the preset (default is the same value as 'tags')",
-      "type": "object",
-      "properties": {},
-      "additionalProperties": true
+      "$ref": "#/definitions/tags"
     },
     "removeTags": {
       "description": "Tags that are removed when changing to another preset (default is the same value as 'addTags' which in turn defaults to 'tags')",
-      "type": "object",
-      "properties": {},
-      "additionalProperties": true
+      "$ref": "#/definitions/tags"
     },
-    "fields": {
+    "fieldIds": {
       "description": "hex-encoded string. IDs of fields to displayed to the user when the preset is created or edited",
       "type": "array",
       "items": {
@@ -61,6 +95,15 @@
       }
     }
   },
-  "required": ["name", "geometry", "tags", "schemaName"],
+  "required": [
+    "name",
+    "geometry",
+    "tags",
+    "addTags",
+    "removeTags",
+    "fieldIds",
+    "schemaName",
+    "terms"
+  ],
   "additionalProperties": false
 }
diff --git a/scripts/generate.js b/scripts/generate.js
@@ -12,6 +12,7 @@ import { generateConfig } from './lib/generate-config.js'
 import { readJSONSchema } from './lib/read-json-schema.js'
 import { generateValidations } from './lib/generate-validations.js'
 import { generateJSONSchemaTS } from './lib/generate-jsonschema-ts.js'
+import { generateEncodeDecode } from './lib/generate-encode-decode.js'
 
 const DIST_DIRNAME = path.join(PROJECT_ROOT, 'dist')
 const TYPES_DIRNAME = path.join(PROJECT_ROOT, 'types')
@@ -32,6 +33,12 @@ fs.writeFileSync(
   protoTypesFile
 )
 
+const encodeDecodeFile = generateEncodeDecode(config)
+fs.writeFileSync(
+  path.join(PROJECT_ROOT, 'types/proto/index.ts'),
+  encodeDecodeFile
+)
+
 const configFile = generateConfig(config)
 fs.writeFileSync(path.join(PROJECT_ROOT, 'config.ts'), configFile)
 

diff --git a/scripts/lib/generate-encode-decode.js b/scripts/lib/generate-encode-decode.js
@@ -0,0 +1,31 @@
+// @ts-check
+/**
+ * @param {ReturnType<import('./parse-config.js').parseConfig>} config
+ */
+export function generateEncodeDecode({ currentSchemaVersions, protoTypeDefs }) {
+  const typeImports = protoTypeDefs.map(
+    ({ schemaName, schemaVersion, typeName }) => {
+      return `import { ${typeName} } from './${schemaName}/v${schemaVersion}'`
+    }
+  )
+
+  const currentProtoTypeDefs = protoTypeDefs.filter(
+    ({ schemaName, schemaVersion }) => {
+      return currentSchemaVersions[schemaName] === schemaVersion
+    }
+  )
+
+  const encodeLines = ['export const Encode = {']
+  for (const { schemaName, typeName } of currentProtoTypeDefs) {
+    encodeLines.push(`  ${schemaName}: ${typeName}.encode,`)
+  }
+  encodeLines.push('}')
+
+  const decodeLines = ['export const Decode = {']
+  for (const { typeName } of protoTypeDefs) {
+    decodeLines.push(`  ${typeName}: ${typeName}.decode,`)
+  }
+  decodeLines.push('}')
+
+  return [...typeImports, '', ...encodeLines, '', ...decodeLines, ''].join('\n')
+}
diff --git a/scripts/lib/generate-proto-types.js b/scripts/lib/generate-proto-types.js
@@ -1,7 +1,5 @@
 // @ts-check
-
-// types/proto/index.d.ts and types/proto/index.js
-import { capitalize } from './utils.js'
+import { getTypeName } from './utils.js'
 
 /**
  * @param {ReturnType<import('./parse-config.js').parseConfig>} config
@@ -22,25 +20,27 @@ export function generateProtoTypes({ currentSchemaVersions, protoTypeDefs }) {
       .join('\n  | ')
 
   const protoTypesWithSchemaInfo =
-    '/** Union of all Proto Types (including non-current versions) with schemaName and schemaVersion */' +
+    '/** Union of all Proto Types (including non-current versions) with schemaName and schemaVersion */\n' +
     'export type ProtoTypesWithSchemaInfo =\n  | ' +
     protoTypeDefs
       .map(({ schemaName, schemaVersion, typeName }) => {
-        return `${typeName} & { schemaName: '${schemaName}', schemaVersion: ${schemaVersion} }`
+        return `(${typeName} & { schemaName: '${schemaName}', schemaVersion: ${schemaVersion} })`
       })
       .join('\n  | ')
 
   const currentProtoTypes =
-    'export type CurrentProtoTypes =\n  | ' +
+    '/** Union of current proto types */\n' +
+    'export type CurrentProtoTypes = {\n' +
     protoTypeDefs
       .filter(({ schemaName, schemaVersion }) => {
         return currentSchemaVersions[schemaName] === schemaVersion
       })
       .map(({ schemaName, schemaVersion }) => {
         const typeName = getTypeName(schemaName, schemaVersion)
-        return `${typeName}`
+        return `  ${schemaName}: ${typeName},`
       })
-      .join('\n  | ')
+      .join('\n') +
+    '}\n'
 
   const protoTypesExports = protoTypeDefs
     .map(({ typeName }) => {
@@ -49,6 +49,7 @@ export function generateProtoTypes({ currentSchemaVersions, protoTypeDefs }) {
     .join('\n')
 
   const protoTypeNames =
+    '/** Union of all valid names of proto types (`${capitalizedSchemaName}_${schemaVersion}`) */\n' +
     'export type ProtoTypeNames =\n  | ' +
     protoTypeDefs.map(({ typeName }) => `'${typeName}'`).join('\n  | ')
 
@@ -67,7 +68,3 @@ export function generateProtoTypes({ currentSchemaVersions, protoTypeDefs }) {
     '\n'
   )
 }
-
-function getTypeName(schemaName, schemaVersion) {
-  return capitalize(schemaName) + '_' + schemaVersion
-}
diff --git a/scripts/lib/utils.js b/scripts/lib/utils.js
@@ -5,6 +5,10 @@ export function capitalize(str) {
   return str.charAt(0).toUpperCase() + str.slice(1)
 }
 
+export function getTypeName(schemaName, schemaVersion) {
+  return capitalize(schemaName) + '_' + schemaVersion
+}
+
 export const PROJECT_ROOT = path.resolve(
   path.dirname(new URL(import.meta.url).pathname),
   '../..'

diff --git a/src/constants.ts b/src/constants.ts
@@ -0,0 +1,4 @@
+/** Length in bytes of data type ID encoding */
+export const DATA_TYPE_ID_BYTES = 6
+/** Length in bytes of schema version encoding */
+export const SCHEMA_VERSION_BYTES = 2
diff --git a/src/decode.ts b/src/decode.ts
@@ -1,4 +1,4 @@
-import { type ProtoTypeNames, ProtoTypes } from '../types/proto/types'
+import { ProtoTypes } from '../types/proto/types'
 import {
   type JsonSchemaTypes,
   type ProtoTypesWithSchemaInfo,
@@ -8,18 +8,18 @@ import {
   type ValidSchemaDef,
 } from './types'
 
-import * as ProtobufEncodeDecode from '../types/proto/index.mapeo'
+import { Decode } from '../types/proto/index'
 import { dataTypeIds, knownSchemaVersions } from '../config'
 import {
   convertProject,
   convertField,
   convertObservation,
+  convertPreset,
 } from './lib/decode-conversions'
 // @ts-ignore
 import * as cenc from 'compact-encoding'
-
-const dataTypeIdSize = 6
-const schemaVersionSize = 2
+import { DATA_TYPE_ID_BYTES, SCHEMA_VERSION_BYTES } from './constants'
+import { getProtoTypeName } from './lib/utils'
 
 /** Map of dataTypeIds to schema names for quick lookups */
 const dataTypeIdToSchemaName: Record<string, SchemaName> = {}
@@ -39,14 +39,14 @@ export function decode(buf: Buffer, versionObj: VersionObj): JsonSchemaTypes {
   const schemaDef = decodeBlockPrefix(buf)
 
   const encodedMsg = buf.subarray(
-    dataTypeIdSize + schemaVersionSize,
+    DATA_TYPE_ID_BYTES + SCHEMA_VERSION_BYTES,
     buf.length
   )
 
-  const messageWithSchemaInfo =
-    ProtobufEncodeDecode[getProtoTypeName(schemaDef)].decode(encodedMsg)
+  const messageWithoutSchemaInfo =
+    Decode[getProtoTypeName(schemaDef)](encodedMsg)
 
-  const message = mutatingSetSchemaDef(messageWithSchemaInfo, schemaDef)
+  const message = mutatingSetSchemaDef(messageWithoutSchemaInfo, schemaDef)
 
   switch (message.schemaName) {
     case 'project':
@@ -55,6 +55,8 @@ export function decode(buf: Buffer, versionObj: VersionObj): JsonSchemaTypes {
       return convertObservation(message, versionObj)
     case 'field':
       return convertField(message, versionObj)
+    case 'preset':
+      return convertPreset(message, versionObj)
     default:
       const _exhaustiveCheck: never = message
       return message
@@ -67,21 +69,21 @@ export function decode(buf: Buffer, versionObj: VersionObj): JsonSchemaTypes {
  * Will throw if dataTypeId and schema version is unknown
  */
 export function decodeBlockPrefix(buf: Buffer): ValidSchemaDef {
-  if (buf.length < dataTypeIdSize + schemaVersionSize) {
+  if (buf.length < DATA_TYPE_ID_BYTES + SCHEMA_VERSION_BYTES) {
     throw new Error('Invalid block prefix - unexpected prefix length')
   }
   const state = cenc.state()
   state.buffer = buf
   state.start = 0
-  state.end = dataTypeIdSize
-  const dataTypeId = cenc.hex.fixed(dataTypeIdSize).decode(state)
+  state.end = DATA_TYPE_ID_BYTES
+  const dataTypeId = cenc.hex.fixed(DATA_TYPE_ID_BYTES).decode(state)
 
   if (typeof dataTypeId !== 'string') {
     throw new Error('Invalid block prefix, could not decode dataTypeId')
   }
 
-  state.start = dataTypeIdSize
-  state.end = dataTypeIdSize + schemaVersionSize
+  state.start = DATA_TYPE_ID_BYTES
+  state.end = DATA_TYPE_ID_BYTES + SCHEMA_VERSION_BYTES
   const schemaVersion = cenc.uint16.decode(state)
 
   if (typeof schemaVersion !== 'number') {
@@ -131,20 +133,6 @@ function assertKnownSchemaDef(schemaDef: {
   }
 }
 
-/**
- * Get the name of the type, e.g. `Observation_5` for schemaName `observation`
- * and schemaVersion `1`
- */
-function getProtoTypeName(schemaDef: ValidSchemaDef): ProtoTypeNames {
-  return (capitalize(schemaDef.schemaName) +
-    '_' +
-    schemaDef.schemaVersion) as ProtoTypeNames
-}
-
-function capitalize<T extends string>(str: T): Capitalize<T> {
-  return (str.charAt(0).toUpperCase() + str.slice(1)) as any
-}
-
 // function mutatingOmit<T, K extends keyof any>(obj: T, key: K): OmitUnion<T, K> {
 //   delete (obj as any)[key]
 //   return obj as any